Tag the query words and the listing text with one shared table: colour, material, brand and product type. Every concept must hold, but each may hold in words or in derived fields: a listing whose featuresOne measured property of a listing for a query, for example its BM25 score or its age. Features are the inputs of a learned model. say 'color : champagne' satisfies colour CHAMPAGNE. Chapter 4 candidatesAll listings that matched the query at all, before they are ordered. Scoring can only reorder this set. It cannot add to it. are kept, attribute-satisfied listings are added, and a structured match counts 1 while a word match counts half. The table is coarse: it also adds a Moen cartridge kit for 'moen faucet', and a bare word with no table entry, such as 'marble', claims no attribute and stays a word search.
Where to look: match keeps the chapter 4 candidatesAll listings that matched the query at all, before they are ordered. Scoring can only reorder this set. It cannot add to it. and adds listings whose attributes satisfy every query concept. score adds the structured and lexicalAbout the literal words. Lexical matching finds listings that contain the query words themselves, not their meaning. concept signals.
Entity tagging reads brand MOEN and type FAUCET: the Moen faucet whose title never writes 'moen' enters the top 10, although chapter 4 finds nothing for it. The ontology is coarse, so it also adds a Moen cartridge kit — the precision this recall costs.
Require every named attribute, but accept each in words or in listing fields. Chapter 4 candidatesAll listings that matched the query at all, before they are ordered. Scoring can only reorder this set. It cannot add to it. are kept; attribute-satisfied listings are added.
import { levelEntitiesContent } from "../../content/levels/level-entities.ts";import { type ConceptDetail, , type EntityConcept, , , , ,} from "../query-understanding/entities.ts";import { } from "../signals/build-signals.ts";import { } from "../text/item-text.ts";import { } from "../text/.ts";import type { CorpusItem } from "../types/corpus.ts";import type { EvolutionLevel, InvertedIndex, LevelDeps, LevelRanker } from "../types/level.ts";import type { QueryUnderstanding } from "../types/query.ts";import type { CandidateSet, RetrievalSignal } from "../types/ranking.ts";import { as } from "./level-1-spelling.ts";const STRUCTURED_MATCH_WEIGHT = 0.05;const LEXICAL_CONCEPT_WEIGHT = 0.025;/** AND over concepts, OR over representations: keep chapter 4 and add attribute matches. */// Every later level builds this level again as its "previous". The listing concepts depend// only on the index, so they are extracted once per index, not once per level.const conceptsByIndex = new WeakMap<InvertedIndex, ReadonlyMap<string, readonly EntityConcept[]>>();function itemConceptsFor(index: InvertedIndex): ReadonlyMap<string, readonly EntityConcept[]> { const known = .get(index); if (known !== undefined) { return known; } const conceptsByItem = new Map<string, readonly EntityConcept[]>(); for (const entry of index.availableItems) { conceptsByItem.set(entry.id, (entry)); } .set(index, conceptsByItem); return conceptsByItem;}export function createRanker(deps: LevelDeps): LevelRanker { const previous = (deps); const conceptsByItem = (deps.index); const unionByQuery = new WeakMap<QueryUnderstanding, CandidateSet>(); function unionFor(understanding: QueryUnderstanding): CandidateSet { const known = unionByQuery.get(understanding); if (known !== undefined) return known; const union = (deps, conceptsByItem, previous, understanding); unionByQuery.set(understanding, union); return union; } return { match: (understanding) => unionFor(understanding), score: (understanding, candidate) => (conceptsByItem, previous, understanding, candidate), correctionsFor: (understanding) => previous.correctionsFor?.(understanding) ?? [], conceptsFor: (understanding) => (understanding, previous), };}/** Concepts from corrected words, so "iphne 13" tags what "iphone 13" tags. */function queryConcepts(understanding: QueryUnderstanding, previous: LevelRanker) { const fixes = previous.correctionsFor?.(understanding) ?? []; const tokens = (understanding.tokens, fixes); const tagged = tokens === understanding.tokens ? understanding : { ...understanding, tokens }; return (tagged);}function widenCandidates( deps: LevelDeps, conceptsByItem: ReadonlyMap<string, readonly EntityConcept[]>, previous: LevelRanker, understanding: QueryUnderstanding,): CandidateSet { const base = previous.match(understanding); const concepts = (understanding, previous); if (concepts.length === 0) return base; const seen = new Set(base.items.map((entry) => entry.id)); const added: CorpusItem[] = []; for (const entry of deps.index.availableItems) { if (!seen.has(entry.id)) { const known = conceptsByItem.get(entry.id) ?? []; if ((concepts, ((entry)), known)) { added.push(entry); seen.add(entry.id); } } } return { items: [...base.items, ...added] };}function conceptScoring( conceptsByItem: ReadonlyMap<string, readonly EntityConcept[]>, previous: LevelRanker, understanding: QueryUnderstanding, candidate: CorpusItem,): ReturnType<LevelRanker["score"]> { const base = previous.score(understanding, candidate); const concepts = (understanding, previous); const known = conceptsByItem.get(candidate.id) ?? []; const detail = (concepts, ((candidate)), known); return { item: candidate, score: base.score + * detail.structural.length + * detail.lexical.length, signals: ( (understanding, candidate, [ ...base.signals.map((signal) => ({ name: signal.name, value: signal.value, weight: signal.weight, })), { name: "structuredConceptMatch", value: detail.structural.length, weight: , }, { name: "lexicalConceptMatch", value: detail.lexical.length, weight: , }, ]), detail, ), };}/** Name the deciding concepts on the two signals, so a demoted listing explains itself. */function annotate(signals: readonly RetrievalSignal[], detail: ConceptDetail) { const named = { structuredConceptMatch: detail.structural, lexicalConceptMatch: detail.lexical }; return signals.map((signal) => { const concepts = named[signal.name as keyof typeof named]; if (concepts === undefined) return signal; return { ...signal, explanation: `${signal.explanation} (${concepts.join(", ") || "none"})` }; });}