Search Evolution Explorer

Level 4 · Spelling normalization

src/domain/levels/level-1-spelling.ts · 152 lines
Back to search

What if I mistype a word?

Before lookup, rewrite each unknown word to the closest word sellers actually wrote: one edit for words up to 4 letters, two for longer ones, only for words seen in at least two listings, and only when the guess is unique. A word written as one tokenSplit a text into single lower-case words without punctuation. that is two words, such as 'roomtables', is split. The rewrite is shown above the list, so you can see what ran. A few demo typos are additionally pinned in query understandingEverything the engine works out about the query before it searches: the normalised words, added synonyms, a guessed category, a corrected spelling., so every chapter guesses them; chapter 4 is the first that looks the corrected word up. A word with two equally close neighboursA nearby item in the trained vector space. The neighbour budget limits how many of them may enter the candidate list, so validation can trade extra recall against extra cost., or a string far from every word such as 'xqzt', is left alone and still finds nothing.

Where to look: match corrects the tokensSplit a text into single lower-case words without punctuation. and runs the chapter 3 lookup on the corrected words. score reuses the chapter 3 score and adds the spellingCorrection flag.

Guided example

ligth bulbvs chapter 3

Spelling correction rewrites 'ligth' to 'light' before lookup: the dimmable light bulb enters the top 10, while chapter 3 returns nothing for 'ligth bulb' because it looks the typo up as typed.

Run it hereOpen in compare

Prerequisites

Reuses the ranking fromChapter 3
Foundation · Inverted indexbuilt at start
Declared artifactsready
src/domain/levels/level-1-spelling.ts152 linessame module the pipeline ran

Correct the typo before lookup: unknown query words are rewritten to listing words, then chapter 3 runs on the corrected words.

import { levelSpellingContent } from "../../content/levels/level-spelling.ts";
import { } from "../index/inverted-index.ts";
import { } from "../query-understanding/category-from-tokens.ts";
import { } from "../query-understanding/entities.ts";
import { } from "../query-understanding/understand.ts";
import { } from "../signals/build-signals.ts";
import { , } from "../text/analyze-query.ts";
import { } from "../text/item-text.ts";
import { , } from "../text/spell-correction.ts";
import type { EvolutionLevel, InvertedIndex, LevelDeps, LevelRanker } from "../types/level.ts";
import type { LeverConfiguration, QueryUnderstanding, SpellingCorrection } from "../types/query.ts";
import { as } from "./level-1-lexical.ts";
interface CorrectedQuery {
readonly corrected: QueryUnderstanding;
readonly fixes: readonly SpellingCorrection[];
}
/** A suggestion must be an established word: seen in more than one listing. */
const MIN_DOCUMENTS_FOR_CORRECTION = 2;
// Every later level builds this level again as its "previous". The vocabulary depends only on
// the index (built with the same levers), so it is counted once per index, not once per level.
const vocabularyByIndex = new WeakMap<InvertedIndex, ReadonlySet<string>>();
function vocabularyFor(deps: LevelDeps): ReadonlySet<string> {
const known = .get(deps.index);
if (known !== undefined) {
return known;
}
const vocabulary = (
deps.index.availableItems.map((entry) => (entry)),
,
deps.levers.variantRules,
);
.set(deps.index, vocabulary);
return vocabulary;
}
/** Correct the typo first, then run the chapter 3 engine on the corrected words. */
export function createRanker(deps: LevelDeps): LevelRanker {
const previous = (deps);
const vocabulary = (deps);
const correctedByQuery = new WeakMap<QueryUnderstanding, CorrectedQuery>();
function correctedFor(understanding: QueryUnderstanding): CorrectedQuery {
const known = correctedByQuery.get(understanding);
if (known !== undefined) {
return known;
}
const result = (understanding, vocabulary, deps.levers);
correctedByQuery.set(understanding, result);
return result;
}
return {
match: (understanding) => (deps, previous, correctedFor, understanding),
score: (understanding, candidate) =>
(previous, correctedFor, understanding, candidate),
correctionsFor: (understanding) => correctedFor(understanding).fixes,
};
}
function correctedCandidates(
deps: LevelDeps,
previous: LevelRanker,
correctedFor: (understanding: QueryUnderstanding) => CorrectedQuery,
understanding: QueryUnderstanding,
): ReturnType<LevelRanker["match"]> {
previous.match(understanding);
const matching = (correctedFor(understanding).corrected);
return (deps.index, (matching, deps.levers.synonymGroups));
}
function correctedScoring(
previous: LevelRanker,
correctedFor: (understanding: QueryUnderstanding) => CorrectedQuery,
understanding: QueryUnderstanding,
candidate: Parameters<LevelRanker["score"]>[1],
): ReturnType<LevelRanker["score"]> {
const { corrected, fixes } = correctedFor(understanding);
const base = previous.score(corrected, candidate);
return {
item: candidate,
score: base.score,
signals: (corrected, candidate, [
...base.signals.map((signal) => ({
name: signal.name,
value: signal.value,
weight: signal.weight ?? 1,
})),
{ name: "spellingCorrection", value: fixes.length > 0 ? 1 : 0 },
]),
};
}
function applyCorrection(
understanding: QueryUnderstanding,
vocabulary: ReadonlySet<string>,
levers: LeverConfiguration,
): CorrectedQuery {
// The pinned demo typos run first, so a corpus artifact cannot win the edit-distance race.
const pinned = (understanding.tokens, understanding.spellingCorrection);
const { corrected, corrections } = (pinned, vocabulary);
const fixes = (corrections, understanding.spellingCorrection);
if (pinned === understanding.tokens && corrections.length === 0) {
return { corrected: understanding, fixes };
}
const rebuilt = (corrected.join(" "), levers, understanding.leverOverrides ?? {});
return {
corrected: {
...understanding,
tokens: corrected,
normalizedTokens: rebuilt.normalizedTokens,
expandedTokens: rebuilt.expandedTokens,
predictedCategoryId: (corrected) ?? understanding.predictedCategoryId,
spellingCorrection: fixes,
},
fixes,
};
}
function mergeCorrections(
vocabulary: readonly SpellingCorrection[],
hardcoded: readonly SpellingCorrection[],
): SpellingCorrection[] {
const from = new Set(vocabulary.map((fix) => fix.from));
return [...vocabulary, ...hardcoded.filter((fix) => !from.has(fix.from))];
}
a numbered note, explained on the righta function you can open heredefined further down in this filewhere level 3 joins