Search Evolution Explorer

Level 9 · Hybrid search

src/domain/levels/level-4-hybrid.ts · 120 lines
Back to search

Find the meaning, not only the words

A neural networkA model with many adjustable numbers that learns patterns from examples instead of being told the rules. The encoder that turns text into a vector is one. turns each listing into a vectorA list of a few hundred numbers that a neural network produces from a text. Texts with a similar meaning get vectors that point in a similar direction, even when they share no words. once, ahead of time. At query time the query becomes a vector too, and the listings whose vectors point the same way become candidatesAll listings that matched the query at all, before they are ordered. Scoring can only reorder this set. It cannot add to it., even when no word matches. The word-based list and the vector list are merged by rank, because their scores are on different scales.

Where to look: match runs the lexicalAbout the literal words. Lexical matching finds listings that contain the query words themselves, not their meaning. match and the vectorA list of a few hundred numbers that a neural network produces from a text. Texts with a similar meaning get vectors that point in a similar direction, even when they share no words. search and unions the two candidate sets. score fuses the two ranks.

Guided example

area rugvs chapter 8

Hybrid retrieval widens chapter 8's candidate set: the vector list adds the 'tomas chevron rug', which never writes 'area rug', and the fused order lifts it into the top 10.

Run it hereOpen in compare

Prerequisites

Reuses the ranking fromChapter 8
Foundation · Inverted indexbuilt at start
Declared artifactsmissing
npm run data:embeddings
src/domain/levels/level-4-hybrid.ts120 linessame module the pipeline ran

MatchingThe step that decides which listings are candidates at all. changes again: the candidatesAll listings that matched the query at all, before they are ordered. Scoring can only reorder this set. It cannot add to it. are the chapter 8 candidates plus the listings whose vectorsA list of a few hundred numbers that a neural network produces from a text. Texts with a similar meaning get vectors that point in a similar direction, even when they share no words. are closest to the query. The two rankings are fused by position.

import { level4Content } from "../../content/levels/level-4.ts";
import { } from "../fusion/reciprocal-rank-fusion.ts";
import { } from "../signals/build-signals.ts";
import { } from "../sort-score.ts";
import type { CorpusItem } from "../types/corpus.ts";
import type { EvolutionLevel, LevelDeps, LevelRanker } from "../types/level.ts";
import type { QueryUnderstanding } from "../types/query.ts";
import { , } from "../vector/vector-retrieval.ts";
import { as } from "./level-3-learned.ts";
const VECTOR_LIMIT = 50;
const RRF_K = 60;
/** Duplication across levels is accepted so each evolution level stays independently readable. */
export function createRanker(deps: LevelDeps): LevelRanker {
const previous = (deps);
const byId = new Map(deps.index.availableItems.map((item) => [item.id, item]));
const fusionByQuery = new WeakMap<QueryUnderstanding, ReadonlyMap<string, number>>();
return {
match(understanding) {
const scores = (understanding, previous, deps, fusionByQuery);
return { items: [...scores.keys()].flatMap((id) => (byId, id)) };
},
score(understanding, candidate) {
const rrf = (understanding, previous, deps, fusionByQuery).get(candidate.id) ?? 0;
const vectorSimilarity = (understanding.queryVector, candidate.id, deps);
return {
item: candidate,
score: rrf,
signals: (understanding, candidate, [
{ name: "rrf", value: rrf },
{ name: "vectorSimilarity", value: vectorSimilarity },
]),
};
},
};
}
function fusionScores(
understanding: QueryUnderstanding,
previous: LevelRanker,
deps: LevelDeps,
cache: WeakMap<QueryUnderstanding, ReadonlyMap<string, number>>,
): ReadonlyMap<string, number> {
const cached = cache.get(understanding);
if (cached !== undefined) {
return cached;
}
const lexical = previous.match(understanding);
const vectorIds = (understanding.queryVector, deps);
const lexicalRanked = (previous, understanding, lexical.items);
const fused = ([lexicalRanked, vectorIds], );
const scores = new Map(fused.map((row) => [row.itemId, row.score]));
cache.set(understanding, scores);
return scores;
}
function vectorIdsFor(queryVector: Float32Array | null, deps: LevelDeps): string[] {
if (queryVector === null || deps.vectorStore === undefined) {
return [];
}
return (queryVector, deps.vectorStore, ).map((hit) => hit.itemId);
}
function cosineFor(queryVector: Float32Array | null, itemId: string, deps: LevelDeps): number {
if (queryVector === null || deps.vectorStore === undefined) {
return 0;
}
const documentVector = deps.vectorStore.getVector(itemId);
if (documentVector === undefined) {
return 0;
}
return (queryVector, documentVector);
}
function rankLexical(
previous: LevelRanker,
understanding: QueryUnderstanding,
items: readonly CorpusItem[],
): string[] {
return items
.map((item) => ({ id: item.id, score: previous.score(understanding, item).score }))
.sort((left, right) => ([left.id, left.score], [right.id, right.score]))
.map((row) => row.id);
}
function itemOf(byId: ReadonlyMap<string, CorpusItem>, id: string): CorpusItem[] {
const item = byId.get(id);
return item === undefined ? [] : [item];
}
a numbered note, explained on the righta function you can open heredefined further down in this filewhere level 8 joins