match_query: unit test and rewrite for TypeScript (#208)

* match_query: unit test and rewrite for TypeScript

* match_query: use new type for parse errors

* match_query: avoid exceptional control flow in date parsing

assets/js/match_query.ts

@@ -0,0 +1,15 @@
+import { defaultMatcher } from './query/matcher';
+import { generateLexArray } from './query/lex';
+import { parseTokens } from './query/parse';
+import { getAstMatcherForTerm } from './query/term';
+
+function parseWithDefaultMatcher(term: string, fuzz: number) {
+  return getAstMatcherForTerm(term, fuzz, defaultMatcher);
+}
+
+function parseSearch(query: string) {
+  const tokens = generateLexArray(query, parseWithDefaultMatcher);
+  return parseTokens(tokens);
+}
+
+export default parseSearch;

assets/js/query/__tests__/date.spec.ts

@@ -0,0 +1,106 @@
+import { makeDateMatcher } from '../date';
+
+function daysAgo(days: number) {
+  return new Date(Date.now() - days * 86400000).toISOString();
+}
+
+describe('Date parsing', () => {
+  it('should match relative dates (upper bound)', () => {
+    const matcher = makeDateMatcher('3 days ago', 'lte');
+
+    expect(matcher(daysAgo(4), 'created_at', 0)).toBe(true);
+    expect(matcher(daysAgo(2), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match relative dates (lower bound)', () => {
+    const matcher = makeDateMatcher('3 days ago', 'gte');
+
+    expect(matcher(daysAgo(4), 'created_at', 0)).toBe(false);
+    expect(matcher(daysAgo(2), 'created_at', 0)).toBe(true);
+  });
+
+  it('should match absolute date ranges', () => {
+    const ltMatcher = makeDateMatcher('2025', 'lt');
+    const gtMatcher = makeDateMatcher('2023', 'gt');
+
+    expect(ltMatcher(new Date(Date.UTC(2025, 5, 21)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(ltMatcher(new Date(Date.UTC(2024, 5, 21)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(ltMatcher(new Date(Date.UTC(2023, 5, 21)).toISOString(), 'created_at', 0)).toBe(true);
+
+    expect(gtMatcher(new Date(Date.UTC(2025, 5, 21)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(gtMatcher(new Date(Date.UTC(2024, 5, 21)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(gtMatcher(new Date(Date.UTC(2023, 5, 21)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match absolute dates through years', () => {
+    const matcher = makeDateMatcher('2024', 'eq');
+
+    expect(matcher(new Date(Date.UTC(2025, 5, 21)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(matcher(new Date(Date.UTC(2023, 5, 21)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match absolute dates through months', () => {
+    const matcher = makeDateMatcher('2024-06', 'eq');
+
+    expect(matcher(new Date(Date.UTC(2024, 6, 21)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(matcher(new Date(Date.UTC(2024, 4, 21)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match absolute dates through days', () => {
+    const matcher = makeDateMatcher('2024-06-21', 'eq');
+
+    expect(matcher(new Date(Date.UTC(2024, 5, 22)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(matcher(new Date(Date.UTC(2024, 5, 20)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match absolute dates through hours', () => {
+    const matcher = makeDateMatcher('2024-06-21T06', 'eq');
+
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 7)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 6)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 5)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match absolute dates through minutes', () => {
+    const matcher = makeDateMatcher('2024-06-21T06:21', 'eq');
+
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 6, 22)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 6, 21)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 6, 20)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match absolute dates through seconds', () => {
+    const matcher = makeDateMatcher('2024-06-21T06:21:30Z', 'eq');
+
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 6, 21, 31)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 6, 21, 30)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 6, 21, 29)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match absolute dates through seconds with positive timezone offset', () => {
+    const matcher = makeDateMatcher('2024-06-21T06:21:30+01:30', 'eq');
+
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 4, 51, 31)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 4, 51, 30)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 4, 51, 29)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should match absolute dates through seconds with negative timezone offset', () => {
+    const matcher = makeDateMatcher('2024-06-21T06:21:30-01:30', 'eq');
+
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 7, 51, 31)).toISOString(), 'created_at', 0)).toBe(false);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 7, 51, 30)).toISOString(), 'created_at', 0)).toBe(true);
+    expect(matcher(new Date(Date.UTC(2024, 5, 21, 7, 51, 29)).toISOString(), 'created_at', 0)).toBe(false);
+  });
+
+  it('should not match malformed absolute date expressions', () => {
+    expect(() => makeDateMatcher('2024-06-21T06:21:30+01:3020', 'eq')).toThrow('Cannot parse date string: 2024-06-21T06:21:30+01:3020');
+  });
+
+  it('should not match malformed relative date expressions', () => {
+    expect(() => makeDateMatcher('3 test failures ago', 'eq')).toThrow('Cannot parse date string: 3 test failures ago');
+  });
+});

assets/js/query/__tests__/lex.spec.ts

@@ -0,0 +1,177 @@
+import { generateLexArray } from '../lex';
+import { AstMatcher } from '../types';
+
+describe('Lexical analysis', () => {
+  let terms: string[];
+  let fuzzes: number[];
+  let boosts: number[];
+
+  function noMatch() {
+    return false;
+  }
+
+  function parseTerm(term: string, fuzz: number, boost: number): AstMatcher {
+    terms.push(term);
+    fuzzes.push(fuzz);
+    boosts.push(boost);
+
+    return noMatch;
+  }
+
+  beforeEach(() => {
+    terms = [];
+    fuzzes = [];
+    boosts = [];
+  });
+
+  it('should lex single terms', () => {
+    const array = generateLexArray('safe', parseTerm);
+    expect(terms).toEqual(['safe']);
+    expect(fuzzes).toEqual([0]);
+    expect(boosts).toEqual([1]);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should lex single terms with fuzzing', () => {
+    const array = generateLexArray('safe~4', parseTerm);
+    expect(terms).toEqual(['safe']);
+    expect(fuzzes).toEqual([4]);
+    expect(boosts).toEqual([1]);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should lex single terms with boosting', () => {
+    const array = generateLexArray('safe^2', parseTerm);
+    expect(terms).toEqual(['safe']);
+    expect(fuzzes).toEqual([0]);
+    expect(boosts).toEqual([2]);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should lex quoted single terms', () => {
+    const array = generateLexArray('"safe"', parseTerm);
+    expect(terms).toEqual(['"safe"']);
+    expect(fuzzes).toEqual([0]);
+    expect(boosts).toEqual([1]);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should lex multiple terms connected by AND', () => {
+    const array = generateLexArray('safe AND solo', parseTerm);
+    expect(terms).toEqual(['safe', 'solo']);
+    expect(fuzzes).toEqual([0, 0]);
+    expect(boosts).toEqual([1, 1]);
+    expect(array).toEqual([noMatch, noMatch, 'and_op']);
+  });
+
+  it('should lex multiple terms connected by OR', () => {
+    const array = generateLexArray('safe OR solo', parseTerm);
+    expect(terms).toEqual(['safe', 'solo']);
+    expect(fuzzes).toEqual([0, 0]);
+    expect(boosts).toEqual([1, 1]);
+    expect(array).toEqual([noMatch, noMatch, 'or_op']);
+  });
+
+  it('should prioritize AND over OR', () => {
+    const array = generateLexArray('safe OR solo AND fluttershy', parseTerm);
+    expect(terms).toEqual(['safe', 'solo', 'fluttershy']);
+    expect(array).toEqual([noMatch, noMatch, noMatch, 'and_op', 'or_op']);
+  });
+
+  it('should override ordering when using parenthetical expressions', () => {
+    const array = generateLexArray('(safe OR solo) AND fluttershy', parseTerm);
+    expect(terms).toEqual(['safe', 'solo', 'fluttershy']);
+    expect(fuzzes).toEqual([0, 0, 0]);
+    expect(boosts).toEqual([1, 1, 1]);
+    expect(array).toEqual([noMatch, noMatch, 'or_op', noMatch, 'and_op']);
+  });
+
+  it('should lex unary NOT', () => {
+    const array = generateLexArray('NOT safe', parseTerm);
+    expect(terms).toEqual(['safe']);
+    expect(array).toEqual([noMatch, 'not_op']);
+  });
+
+  it('should prioritize NOT over AND', () => {
+    const array = generateLexArray('NOT safe AND solo', parseTerm);
+    expect(terms).toEqual(['safe', 'solo']);
+    expect(array).toEqual([noMatch, 'not_op', noMatch, 'and_op']);
+  });
+
+  it('should prioritize NOT over OR', () => {
+    const array = generateLexArray('NOT safe OR solo', parseTerm);
+    expect(terms).toEqual(['safe', 'solo']);
+    expect(array).toEqual([noMatch, 'not_op', noMatch, 'or_op']);
+  });
+
+  it('should allow group negation', () => {
+    const array = generateLexArray('NOT (safe OR solo)', parseTerm);
+    expect(terms).toEqual(['safe', 'solo']);
+    expect(array).toEqual([noMatch, noMatch, 'or_op', 'not_op']);
+  });
+
+  it('should allow NOT expressions inside terms', () => {
+    const array = generateLexArray('this NOT that', parseTerm);
+    expect(terms).toEqual(['this NOT that']);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should allow parenthetical expressions inside terms', () => {
+    const array = generateLexArray('rose (flower)', parseTerm);
+    expect(terms).toEqual(['rose (flower)']);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should handle fuzz expressions in place of terms', () => {
+    const array = generateLexArray('~2', parseTerm);
+    expect(terms).toEqual(['~2']);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should handle boost expressions in place of terms', () => {
+    const array = generateLexArray('^2', parseTerm);
+    expect(terms).toEqual(['^2']);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should handle fuzz expressions in terms', () => {
+    const array = generateLexArray('two~2~two', parseTerm);
+    expect(terms).toEqual(['two~2~two']);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should handle boost expressions in terms', () => {
+    const array = generateLexArray('two^2^two', parseTerm);
+    expect(terms).toEqual(['two^2^two']);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should handle quotes in terms', () => {
+    const array = generateLexArray('a "quoted" expression', parseTerm);
+    expect(terms).toEqual(['a "quoted" expression']);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should allow extra spaces in terms', () => {
+    const array = generateLexArray('twilight  sparkle', parseTerm);
+    expect(terms).toEqual(['twilight  sparkle']);
+    expect(array).toEqual([noMatch]);
+  });
+
+  it('should collapse consecutive AND expressions', () => {
+    const array = generateLexArray('safe AND solo AND fluttershy AND applejack', parseTerm);
+    expect(terms).toEqual(['safe', 'solo', 'fluttershy', 'applejack']);
+    expect(array).toEqual([noMatch, noMatch, 'and_op', noMatch, 'and_op', noMatch, 'and_op']);
+  });
+
+  it('should collapse consecutive OR expressions', () => {
+    const array = generateLexArray('safe OR solo OR fluttershy OR applejack', parseTerm);
+    expect(terms).toEqual(['safe', 'solo', 'fluttershy', 'applejack']);
+    expect(array).toEqual([noMatch, noMatch, 'or_op', noMatch, 'or_op', noMatch, 'or_op']);
+  });
+
+  it('should throw exception on mismatched parentheses', () => {
+    expect(() => generateLexArray('(safe OR solo AND fluttershy', parseTerm)).toThrow('Mismatched parentheses.');
+    // expect(() => generateLexArray(')bad', parseTerm)).toThrow('Mismatched parentheses.');
+  });
+});

assets/js/query/__tests__/literal.spec.ts

@@ -0,0 +1,36 @@
+import { makeLiteralMatcher } from '../literal';
+
+describe('Literal field parsing', () => {
+  it('should handle exact matching in arrayed fields', () => {
+    const matcher = makeLiteralMatcher('safe', 0, false);
+    expect(matcher('safe, solo', 'tags', 0)).toBe(true);
+    expect(matcher('solo', 'tags', 0)).toBe(false);
+  });
+
+  it('should handle exact matching in non-arrayed fields', () => {
+    const matcher = makeLiteralMatcher('safe', 0, false);
+    expect(matcher('safe, solo', 'description', 0)).toBe(false);
+    expect(matcher('safe', 'description', 0)).toBe(true);
+    expect(matcher('solo', 'description', 0)).toBe(false);
+  });
+
+  it('should handle fuzzy matching based on normalized edit distance', () => {
+    const matcher = makeLiteralMatcher('fluttersho', 0.8, false);
+    expect(matcher('fluttershy', 'tags', 0)).toBe(true);
+    expect(matcher('rarity', 'tags', 0)).toBe(false);
+  });
+
+  it('should handle fuzzy matching based on raw edit distance', () => {
+    const matcher = makeLiteralMatcher('fluttersho', 1, false);
+    expect(matcher('fluttershy', 'tags', 0)).toBe(true);
+    expect(matcher('rarity', 'tags', 0)).toBe(false);
+  });
+
+  it('should handle wildcard matching', () => {
+    const matcher = makeLiteralMatcher('fl?tter*', 0, true);
+    expect(matcher('fluttershy', 'tags', 0)).toBe(true);
+    expect(matcher('flitter', 'tags', 0)).toBe(true);
+    expect(matcher('rainbow dash', 'tags', 0)).toBe(false);
+    expect(matcher('gentle flutter', 'tags', 0)).toBe(false);
+  });
+});

assets/js/query/__tests__/number.spec.ts

@@ -0,0 +1,53 @@
+import { makeNumberMatcher } from '../number';
+
+describe('Number parsing', () => {
+  it('should match numbers directly', () => {
+    const intMatch = makeNumberMatcher(2067, 0, 'eq');
+
+    expect(intMatch('2066', 'value', 0)).toBe(false);
+    expect(intMatch('2067', 'value', 0)).toBe(true);
+    expect(intMatch('2068', 'value', 0)).toBe(false);
+    expect(intMatch('20677', 'value', 0)).toBe(false);
+  });
+
+  it('should match number ranges', () => {
+    const ltMatch = makeNumberMatcher(2067, 0, 'lt');
+    const lteMatch = makeNumberMatcher(2067, 0, 'lte');
+    const gtMatch = makeNumberMatcher(2067, 0, 'gt');
+    const gteMatch = makeNumberMatcher(2067, 0, 'gte');
+
+    expect(ltMatch('2066', 'value', 0)).toBe(true);
+    expect(ltMatch('2067', 'value', 0)).toBe(false);
+    expect(ltMatch('2068', 'value', 0)).toBe(false);
+    expect(lteMatch('2066', 'value', 0)).toBe(true);
+    expect(lteMatch('2067', 'value', 0)).toBe(true);
+    expect(lteMatch('2068', 'value', 0)).toBe(false);
+    expect(gtMatch('2066', 'value', 0)).toBe(false);
+    expect(gtMatch('2067', 'value', 0)).toBe(false);
+    expect(gtMatch('2068', 'value', 0)).toBe(true);
+    expect(gteMatch('2066', 'value', 0)).toBe(false);
+    expect(gteMatch('2067', 'value', 0)).toBe(true);
+    expect(gteMatch('2068', 'value', 0)).toBe(true);
+  });
+
+  it('should not match unparsed values', () => {
+    const matcher = makeNumberMatcher(2067, 0, 'eq');
+
+    expect(matcher('NaN', 'value', 0)).toBe(false);
+    expect(matcher('test', 'value', 0)).toBe(false);
+  });
+
+  it('should interpret fuzz as an inclusive range around the value', () => {
+    const matcher = makeNumberMatcher(2067, 3, 'eq');
+
+    expect(matcher('2063', 'value', 0)).toBe(false);
+    expect(matcher('2064', 'value', 0)).toBe(true);
+    expect(matcher('2065', 'value', 0)).toBe(true);
+    expect(matcher('2066', 'value', 0)).toBe(true);
+    expect(matcher('2067', 'value', 0)).toBe(true);
+    expect(matcher('2068', 'value', 0)).toBe(true);
+    expect(matcher('2069', 'value', 0)).toBe(true);
+    expect(matcher('2070', 'value', 0)).toBe(true);
+    expect(matcher('2071', 'value', 0)).toBe(false);
+  });
+});