From dd11d3b5ae99a0c95dd3c95bb2c6246f5a414893 Mon Sep 17 00:00:00 2001 From: ascariandrea Date: Fri, 7 Oct 2022 08:39:53 +0200 Subject: [PATCH] build(guardoni): fix cli testing scripts --- packages/shared/src/utils/csv.utils.ts | 50 +++++++ packages/shared/src/utils/fp.utils.ts | 6 +- .../guardoni/__tests__/cli/cli-tk.spec.ts | 2 +- .../guardoni/__tests__/cli/cli-yt.spec.ts | 2 +- platforms/guardoni/__tests__/guardoni.spec.ts | 2 +- platforms/guardoni/experiments/yt-videos.csv | 3 + platforms/guardoni/package.json | 5 +- .../scripts/cli-test-from-fixtures.mjs | 130 ++++++++++++++++++ platforms/guardoni/scripts/cli-tk-test.mjs | 3 +- .../guardoni/scripts/cli-yt-test-home.mjs | 8 +- .../guardoni/scripts/cli-yt-test-videos.mjs | 4 +- platforms/guardoni/src/guardoni/experiment.ts | 5 +- platforms/guardoni/src/guardoni/extension.ts | 12 +- platforms/guardoni/src/guardoni/utils.ts | 44 ------ ...44b8476d67668f19cc9e8d4f235890181d26a.json | 0 ...a43574215629696bccb876a6f7660d3dc2dc9.json | 0 ...b7e93d62ad6fdde6630c34c5ecfe7720474bb.json | 0 .../tktrex/backend/__tests__/native.e2e.ts | 2 +- .../tktrex/backend/__tests__/profile.e2e.ts | 2 +- .../tktrex/backend/__tests__/search.e2e.ts | 2 +- platforms/yttrex/shared/src/models/HTML.ts | 2 +- scripts/build.sh | 3 +- scripts/test-guardoni.sh | 14 ++ scripts/test-tk.sh | 41 ------ 24 files changed, 230 insertions(+), 112 deletions(-) create mode 100644 packages/shared/src/utils/csv.utils.ts create mode 100755 platforms/guardoni/scripts/cli-test-from-fixtures.mjs rename platforms/tktrex/backend/__tests__/fixtures/{ => htmls}/native/1c144b8476d67668f19cc9e8d4f235890181d26a.json (100%) rename platforms/tktrex/backend/__tests__/fixtures/{ => htmls}/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json (100%) rename platforms/tktrex/backend/__tests__/fixtures/{ => htmls}/search/812b7e93d62ad6fdde6630c34c5ecfe7720474bb.json (100%) create mode 100755 scripts/test-guardoni.sh delete mode 100755 scripts/test-tk.sh diff --git a/packages/shared/src/utils/csv.utils.ts b/packages/shared/src/utils/csv.utils.ts new file mode 100644 index 000000000..7cb86e4cc --- /dev/null +++ b/packages/shared/src/utils/csv.utils.ts @@ -0,0 +1,50 @@ +import { AppError, toAppError } from '../errors/AppError'; +import csvParse from 'csv-parse'; +import * as csvStringify from 'csv-stringify'; +import * as TE from 'fp-ts/lib/TaskEither'; +import { GetLogger } from '../logger'; +const csvLogger = GetLogger('csv') + +export const csvParseTE = ( + content: Buffer, + options: csvParse.Options +): TE.TaskEither< + AppError, + { + records: any; + info: csvParse.Info; + } +> => + TE.tryCatch( + () => + new Promise((resolve, reject) => { + csvParse(content, options, (error, records, info) => { + if (error) { + csvLogger.error('CSV Parse error: %O', error); + return reject(error); + } + csvLogger.debug('CSV Parse results: %O', records); + return resolve({ records, info }); + }); + }), + toAppError + ); + +export const csvStringifyTE = ( + records: any[], + options: csvStringify.Options +): TE.TaskEither => + TE.tryCatch( + () => + new Promise((resolve, reject) => { + csvStringify.stringify(records, options, (error, info) => { + if (error) { + csvLogger.error('CSV Stringify error: %O', error); + return reject(error); + } + csvLogger.debug('CSV Stringify results: %O', records); + return resolve(info); + }); + }), + toAppError + ); diff --git a/packages/shared/src/utils/fp.utils.ts b/packages/shared/src/utils/fp.utils.ts index eb27af1b9..ceea73e59 100644 --- a/packages/shared/src/utils/fp.utils.ts +++ b/packages/shared/src/utils/fp.utils.ts @@ -10,7 +10,11 @@ export const foldTEOrThrow = (te: TE.TaskEither): Promise => { return pipe( te, TE.fold( - (e) => () => Promise.reject(e), + (e) => () => { + // eslint-disable-next-line + console.error(e); + return Promise.reject(e) + }, (a) => () => Promise.resolve(a) ) )(); diff --git a/platforms/guardoni/__tests__/cli/cli-tk.spec.ts b/platforms/guardoni/__tests__/cli/cli-tk.spec.ts index 1ba0f3389..d089e0004 100644 --- a/platforms/guardoni/__tests__/cli/cli-tk.spec.ts +++ b/platforms/guardoni/__tests__/cli/cli-tk.spec.ts @@ -10,7 +10,7 @@ import differenceInMilliseconds from 'date-fns/differenceInMilliseconds'; import * as fs from 'fs'; import * as path from 'path'; import { GetGuardoniCLI, GuardoniCLI } from '../../src/guardoni/cli'; -import { csvStringifyTE } from '../../src/guardoni/utils'; +import { csvStringifyTE } from '@shared/utils/csv.utils'; import axiosMock from '@shared/test/__mocks__/axios.mock'; import { puppeteerMock } from '@shared/test/__mocks__/puppeteer.mock'; import { formatExperimentList } from '../../src/guardoni/experiment'; diff --git a/platforms/guardoni/__tests__/cli/cli-yt.spec.ts b/platforms/guardoni/__tests__/cli/cli-yt.spec.ts index ed80a8a3e..250925188 100644 --- a/platforms/guardoni/__tests__/cli/cli-yt.spec.ts +++ b/platforms/guardoni/__tests__/cli/cli-yt.spec.ts @@ -10,7 +10,7 @@ import { pipe } from 'fp-ts/lib/function'; import * as fs from 'fs'; import * as path from 'path'; import { GetGuardoniCLI, GuardoniCLI } from '../../src/guardoni/cli'; -import { csvStringifyTE } from '../../src/guardoni/utils'; +import { csvStringifyTE } from '@shared/utils/csv.utils'; import axiosMock from '@shared/test/__mocks__/axios.mock'; import { puppeteerMock } from '@shared/test/__mocks__/puppeteer.mock'; import { getProfileDataDir, readProfile } from '../../src/guardoni/profile'; diff --git a/platforms/guardoni/__tests__/guardoni.spec.ts b/platforms/guardoni/__tests__/guardoni.spec.ts index a8240f455..641cb4540 100644 --- a/platforms/guardoni/__tests__/guardoni.spec.ts +++ b/platforms/guardoni/__tests__/guardoni.spec.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { readConfigFromPath } from '../src/guardoni/config'; import { GetGuardoni } from '../src/guardoni/guardoni'; import { getDefaultProfile, getProfileDataDir } from '../src/guardoni/profile'; -import { csvStringifyTE } from '../src/guardoni/utils'; +import { csvStringifyTE } from '@shared/utils/csv.utils'; import { guardoniLogger } from '../src/logger'; import { fc } from '@shared/test'; diff --git a/platforms/guardoni/experiments/yt-videos.csv b/platforms/guardoni/experiments/yt-videos.csv index 37b965afa..afe60dae6 100644 --- a/platforms/guardoni/experiments/yt-videos.csv +++ b/platforms/guardoni/experiments/yt-videos.csv @@ -2,3 +2,6 @@ title,url,urltag,watchFor,incrementScrollByPX,totalScroll YouChoose: Customize your Recommendations,https://www.youtube.com/watch?v=3laqegktOuQ,youchoose-recommendations,3s,400,1000 YouChoose.ai | Gain Control on your Recommendations!,https://www.youtube.com/watch?v=ReVMTcRA-E4,youchose-gain-Control,3s,400,1000 with Tracking Exposed: knowledge is freedom — algorithm analysis is for anybody!,https://www.youtube.com/watch?v=SmYuYEhT81c,trex-algorithm,5s,400,1000 +How to Investigate TikTok’s Algorithm,https://www.youtube.com/watch?v=jpb83sxcDdI,trex-algorithm,15s,400,1000 +Scopriamo Tracking.Exposed,https://www.youtube.com/watch?v=VKO9UXy-vlo,trex-alorihtm,15s,400,1000 +facebook tracking exposed: come Facebook condiziona il nostro modo di vedere il mondo ,https://www.youtube.com/watch?v=vDzBNID4CjI,trex-alorihtm,15s,400,1000 diff --git a/platforms/guardoni/package.json b/platforms/guardoni/package.json index f3a98e8ad..be8d7eef6 100644 --- a/platforms/guardoni/package.json +++ b/platforms/guardoni/package.json @@ -23,8 +23,9 @@ "prepack": "yarn build", "check-stealth": "ts-node ./bin/check-stealth.ts", "tdd": "jest --watch --verbose", - "cli-yt-test-videos": "node ./scripts/cli-yt-test-videos.mjs", - "cli-yt-test-home": "node ./scripts/cli-yt-test-home.mjs" + "cli-yt-test-videos": "ts-node ./scripts/cli-yt-test-videos.mjs", + "cli-yt-test-home": "ts-node ./scripts/cli-yt-test-home-fixures.mjs", + "cli-yt-test-from-fixtures": "ts-node ./scripts/cli-yt-test-home-fixures.mjs" }, "bin": { "cli": "./bin/guardoni-cli.js" diff --git a/platforms/guardoni/scripts/cli-test-from-fixtures.mjs b/platforms/guardoni/scripts/cli-test-from-fixtures.mjs new file mode 100755 index 000000000..d52ff99eb --- /dev/null +++ b/platforms/guardoni/scripts/cli-test-from-fixtures.mjs @@ -0,0 +1,130 @@ +#!/usr/bin/env node + +/* eslint-disable camelcase */ + +import { $, os, fetch, fs, path } from 'zx'; +import { normalizePlatform, getGuardoniCliPkgName } from './utils.mjs'; +import dotenv from 'dotenv'; +import assert from 'assert'; +import { + readFixtureJSONPaths, + readFixtureJSON, +} from '../../../packages/shared/build/test/utils/parser.utils.js'; +import { csvStringifyTE } from '../../../packages/shared/build/utils/csv.utils.js'; +import { foldTEOrThrow } from '../../../packages/shared/build/utils/fp.utils.js'; + +dotenv.config({ path: '.env.development' }); + +// eslint-disable-next-line no-void +void (async function () { + const [p, nature, count] = process.argv.slice(2); + console.log({ p, nature, count }); + + if (!p) { + console.error('Platform parameter not present'); + process.exit(1); + } + + if (!nature) { + console.error('Nature parameter not present'); + process.exit(1); + } + + const version = await $`node -p -e "require('./package.json').version"`; + const platform = normalizePlatform(os.type()); + const profile = 'profile-test-99'; + + fs.removeSync(path.resolve(process.cwd(), 'profiles', profile)); + + const cli = `./dist/${getGuardoniCliPkgName( + version.stdout.replace('\n', ''), + platform + )}`; + + const flags = [ + '--basePath=./', + `--executablePath=${process.env.PUPPETEER_EXEC_PATH}`, + '-c=guardoni.config.json', + '--headless=false', + '--verbose', + ]; + + const researchTag = `guardoni-${p}-test-${nature}`; + + const experimentFlags = [ + ...flags, + `--publicKey=${process.env.PUBLIC_KEY}`, + `--secretKey=${process.env.SECRET_KEY}`, + `--researchTag=${researchTag}`, + ]; + + const fixturePaths = readFixtureJSONPaths( + path.resolve( + process.cwd(), + `../${p}trex/backend/__tests__/fixtures/htmls/${nature}` + ) + ); + + const take = parseInt(count, 10) ?? fixturePaths.length; + await $`echo "Taking ${take} fixtures"`; + const fixtures = fixturePaths + .slice(0, take) + .map((f) => readFixtureJSON(f, process.env.PUBLIC_KEY)); + + const sources = fixtures.reduce( + (acc, f, i) => + acc.concat( + f.sources.map((s, ii) => ({ + title: f.metadata.title, + url: s.href, + urltag: `${p}-test-${nature}-${i}-${ii}`, + watchFor: '5s', + incrementScrollByPX: 400, + totalScroll: 1000, + })) + ), + [] + ); + + // reject cookie modal + await $`${cli} ${flags} ${p}-navigate`; + + const experiment = await foldTEOrThrow( + csvStringifyTE(sources, { header: 'true' }) + ); + + const experimentFile = path.resolve( + process.cwd(), + `./experiments/${p}-${nature}-test-from-fixtures-temp.csv` + ); + + fs.writeFileSync(experimentFile, experiment, 'utf-8'); + + const yt_home_experiment_register_out = + await $`${cli} ${flags} ${p}-register ${experimentFile}`; + + const yt_home_experiment_id = yt_home_experiment_register_out.stdout + .split('\n') + .find((s) => s.startsWith('experimentId:')) + .replace('experimentId: \t', '') + .trim(); + + const yt_home_experiment_run_out = + await $`${cli} ${experimentFlags} ${p}-experiment ${yt_home_experiment_id} | grep 'publicKey: ' `; + + const yt_home_experiment_public_key = yt_home_experiment_run_out.stdout + .replace('publicKey: \t ', '') + .replace('\n', ''); + + assert.strictEqual(yt_home_experiment_public_key, process.env.PUBLIC_KEY); + + const backend = p === 'tk' ? process.env.TK_BACKEND : process.env.YT_BACKEND; + const personalURL = `${backend}/v2/personal/${yt_home_experiment_public_key}/${nature}/json`; + await $`echo ${personalURL}`; + + // const metadata = await fetch( + // `http://localhost:9000/api/v2/metadata?publicKey=${yt_home_experiment_public_key}&experimentId=${yt_home_experiment_id}&nature=home` + // ).then((r) => r.json()); + + fs.removeSync(experimentFile); +})(); diff --git a/platforms/guardoni/scripts/cli-tk-test.mjs b/platforms/guardoni/scripts/cli-tk-test.mjs index d13e9f044..4a4d348da 100755 --- a/platforms/guardoni/scripts/cli-tk-test.mjs +++ b/platforms/guardoni/scripts/cli-tk-test.mjs @@ -60,7 +60,8 @@ void (async function () { const tk_search_experiment_public_key = tk_search_experiment_run_out.stdout .replace('publicKey: \t', '') - .replace('\n', ''); + .replace('\n', '') + .trim(); await $`echo ${tk_search_experiment_public_key}`; diff --git a/platforms/guardoni/scripts/cli-yt-test-home.mjs b/platforms/guardoni/scripts/cli-yt-test-home.mjs index e0ce638bc..531cc2efe 100755 --- a/platforms/guardoni/scripts/cli-yt-test-home.mjs +++ b/platforms/guardoni/scripts/cli-yt-test-home.mjs @@ -25,7 +25,7 @@ void (async function () { '--basePath=./', `--executablePath=${process.env.PUPPETEER_EXEC_PATH}`, '-c=guardoni.config.json', - '--headless', + '--headless=false', '--verbose', ]; @@ -35,11 +35,11 @@ void (async function () { ...flags, `--publicKey=${process.env.PUBLIC_KEY}`, `--secretKey=${process.env.SECRET_KEY}`, - `--researchTag=${researchTag}` + `--researchTag=${researchTag}`, ]; // reject cookie modal - await $`${cli} ${flags} yt-navigate --cookie-modal=reject --exit --headless=false`; + await $`${cli} ${flags} yt-navigate --cookie-modal=reject --exit`; const yt_home_experiment_register_out = await $`${cli} ${flags} yt-register ./experiments/yt-home.csv | grep 'experimentId: '`; @@ -64,7 +64,7 @@ void (async function () { assert.strictEqual(yt_home_experiment_public_key, process.env.PUBLIC_KEY); const metadata = await fetch( - `http://localhost:9000/api/v2/metadata?experimentId=${yt_home_experiment_id}` + `http://localhost:9000/api/v2/metadata?publicKey=${yt_home_experiment_public_key}&experimentId=${yt_home_experiment_id}` ).then((r) => r.json()); assert.strictEqual(metadata[0].experimentId, yt_home_experiment_id); diff --git a/platforms/guardoni/scripts/cli-yt-test-videos.mjs b/platforms/guardoni/scripts/cli-yt-test-videos.mjs index 476df652f..a01a6d9b9 100755 --- a/platforms/guardoni/scripts/cli-yt-test-videos.mjs +++ b/platforms/guardoni/scripts/cli-yt-test-videos.mjs @@ -24,7 +24,7 @@ void (async function () { '--basePath=./', `--executablePath=${process.env.PUPPETEER_EXEC_PATH}`, '-c=guardoni.config.json', - '--headless', + '--headless=false', '--verbose', ]; @@ -58,7 +58,7 @@ void (async function () { .replace('publicKey: \t ', '') .replace('\n', ''); - assert.strictEqual(ytVideoExperimentPubKey, process.env.PUBLIC_KEY); + // assert.strictEqual(ytVideoExperimentPubKey, process.env.PUBLIC_KEY); const metadata = await fetch( `http://localhost:9000/api/v2/metadata?experimentId=${yt_video_experiment_id}` diff --git a/platforms/guardoni/src/guardoni/experiment.ts b/platforms/guardoni/src/guardoni/experiment.ts index d94acfc1f..531295a71 100644 --- a/platforms/guardoni/src/guardoni/experiment.ts +++ b/platforms/guardoni/src/guardoni/experiment.ts @@ -17,6 +17,7 @@ import _ from 'lodash'; // import * as NEA from 'fp-ts/lib/NonEmptyArray'; import { parseClickCommand } from '@shared/providers/puppeteer/steps/click'; import { parseKeypressCommand } from '@shared/providers/puppeteer/steps/keyPress'; +import { csvParseTE } from '@shared/utils/csv.utils'; import { NonEmptyArray } from 'fp-ts/lib/NonEmptyArray'; import * as TE from 'fp-ts/lib/TaskEither'; import * as fs from 'fs'; @@ -28,9 +29,9 @@ import { ExperimentInfo, GuardoniContext, GuardoniProfile, - GuardoniSuccessOutput, + GuardoniSuccessOutput } from './types'; -import { csvParseTE, liftFromIOE } from './utils'; +import { liftFromIOE } from './utils'; export const validateNonEmptyString = ( s: string diff --git a/platforms/guardoni/src/guardoni/extension.ts b/platforms/guardoni/src/guardoni/extension.ts index baf87b805..409e0611b 100644 --- a/platforms/guardoni/src/guardoni/extension.ts +++ b/platforms/guardoni/src/guardoni/extension.ts @@ -196,16 +196,14 @@ const getSettingsJSONPath = (ctx: GuardoniContext): string => export const setLocalSettings = (ctx: GuardoniContext) => (s?: Partial): void => { - if (!s?.publicKey && !s?.secretKey) { - ctx.logger.debug('No publicKey/secretKey pair given...'); + const keys = Object.keys(s ?? {}); + const settingsJsonPath = getSettingsJSONPath(ctx); + if (keys.length === 0) { + ctx.logger.debug('No values for %s given...', settingsJsonPath); return; } - const settingsJsonPath = getSettingsJSONPath(ctx); - + ctx.logger.info('Saving settings at %s: %O', settingsJsonPath, s); const settings = JSON.stringify(s); - - ctx.logger.info('Saving settings at %s: %O', settingsJsonPath, settings); - fs.writeFileSync(settingsJsonPath, settings); }; diff --git a/platforms/guardoni/src/guardoni/utils.ts b/platforms/guardoni/src/guardoni/utils.ts index d227d78bb..2e20aff44 100644 --- a/platforms/guardoni/src/guardoni/utils.ts +++ b/platforms/guardoni/src/guardoni/utils.ts @@ -1,6 +1,4 @@ import { AppError, toAppError } from '@shared/errors/AppError'; -import csvParse from 'csv-parse'; -import * as csvStringify from 'csv-stringify'; import * as E from 'fp-ts/lib/Either'; import { pipe } from 'fp-ts/lib/function'; import * as IOE from 'fp-ts/lib/IOEither'; @@ -70,49 +68,7 @@ export const toGuardoniSuccessOutput = ( }; }; -export const csvParseTE = ( - content: Buffer, - options: csvParse.Options -): TE.TaskEither< - AppError, - { - records: any; - info: csvParse.Info; - } -> => - TE.tryCatch( - () => - new Promise((resolve, reject) => { - csvParse(content, options, (error, records, info) => { - if (error) { - guardoniLogger.error('CSV Parse error: %O', error); - return reject(error); - } - guardoniLogger.debug('CSV Parse results: %O', records); - return resolve({ records, info }); - }); - }), - toAppError - ); -export const csvStringifyTE = ( - records: any[], - options: csvStringify.Options -): TE.TaskEither => - TE.tryCatch( - () => - new Promise((resolve, reject) => { - csvStringify.stringify(records, options, (error, info) => { - if (error) { - guardoniLogger.error('CSV Stringify error: %O', error); - return reject(error); - } - guardoniLogger.debug('CSV Stringify results: %O', records); - return resolve(info); - }); - }), - toAppError - ); export const liftFromIOE = (lazyF: () => T): TE.TaskEither => { return pipe(IOE.tryCatch(lazyF, toAppError), TE.fromIOEither); diff --git a/platforms/tktrex/backend/__tests__/fixtures/native/1c144b8476d67668f19cc9e8d4f235890181d26a.json b/platforms/tktrex/backend/__tests__/fixtures/htmls/native/1c144b8476d67668f19cc9e8d4f235890181d26a.json similarity index 100% rename from platforms/tktrex/backend/__tests__/fixtures/native/1c144b8476d67668f19cc9e8d4f235890181d26a.json rename to platforms/tktrex/backend/__tests__/fixtures/htmls/native/1c144b8476d67668f19cc9e8d4f235890181d26a.json diff --git a/platforms/tktrex/backend/__tests__/fixtures/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json b/platforms/tktrex/backend/__tests__/fixtures/htmls/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json similarity index 100% rename from platforms/tktrex/backend/__tests__/fixtures/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json rename to platforms/tktrex/backend/__tests__/fixtures/htmls/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json diff --git a/platforms/tktrex/backend/__tests__/fixtures/search/812b7e93d62ad6fdde6630c34c5ecfe7720474bb.json b/platforms/tktrex/backend/__tests__/fixtures/htmls/search/812b7e93d62ad6fdde6630c34c5ecfe7720474bb.json similarity index 100% rename from platforms/tktrex/backend/__tests__/fixtures/search/812b7e93d62ad6fdde6630c34c5ecfe7720474bb.json rename to platforms/tktrex/backend/__tests__/fixtures/htmls/search/812b7e93d62ad6fdde6630c34c5ecfe7720474bb.json diff --git a/platforms/tktrex/backend/__tests__/native.e2e.ts b/platforms/tktrex/backend/__tests__/native.e2e.ts index 763f68746..49dc81ef4 100644 --- a/platforms/tktrex/backend/__tests__/native.e2e.ts +++ b/platforms/tktrex/backend/__tests__/native.e2e.ts @@ -56,7 +56,7 @@ describe('Parser: "native"', () => { jest.setTimeout(20 * 1000); const history = readFixtureJSONPaths( - path.resolve(__dirname, 'fixtures/native') + path.resolve(__dirname, 'fixtures/htmls/native') ); axiosMock.get.mockImplementation((url, config) => { diff --git a/platforms/tktrex/backend/__tests__/profile.e2e.ts b/platforms/tktrex/backend/__tests__/profile.e2e.ts index 49ddfa521..9ae9f1af4 100644 --- a/platforms/tktrex/backend/__tests__/profile.e2e.ts +++ b/platforms/tktrex/backend/__tests__/profile.e2e.ts @@ -57,7 +57,7 @@ describe('Parser: "profile"', () => { jest.setTimeout(20 * 1000); const history = readFixtureJSONPaths( - path.resolve(__dirname, 'fixtures/profile') + path.resolve(__dirname, 'fixtures/htmls/profile') ); axiosMock.get.mockImplementation((url, config) => { diff --git a/platforms/tktrex/backend/__tests__/search.e2e.ts b/platforms/tktrex/backend/__tests__/search.e2e.ts index 0e091f163..1fbaace0d 100644 --- a/platforms/tktrex/backend/__tests__/search.e2e.ts +++ b/platforms/tktrex/backend/__tests__/search.e2e.ts @@ -56,7 +56,7 @@ describe('Parser: "search"', () => { jest.setTimeout(20 * 1000); const history = readFixtureJSONPaths( - path.resolve(__dirname, 'fixtures/search') + path.resolve(__dirname, 'fixtures/htmls/search') ); axiosMock.get.mockImplementation((url, config) => { diff --git a/platforms/yttrex/shared/src/models/HTML.ts b/platforms/yttrex/shared/src/models/HTML.ts index 1a1ea390e..6ca806fff 100644 --- a/platforms/yttrex/shared/src/models/HTML.ts +++ b/platforms/yttrex/shared/src/models/HTML.ts @@ -18,7 +18,7 @@ export const HTML = t.strict( processed: t.boolean, timelineId: t.union([t.string, t.undefined]), n: t.union([t.array(t.any), t.undefined]), - experimentId: t.union([t.string, t.undefined]), + experimentId: t.union([UUID, t.undefined]), researchTag: t.union([t.string, t.undefined]), }, 'HTML' diff --git a/scripts/build.sh b/scripts/build.sh index bad35e10c..1d3881700 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -22,4 +22,5 @@ yarn tk:ext build # build guardoni yarn guardoni build:cli -# yarn guardoni pkg +yarn guardoni build:app +yarn guardoni pkg diff --git a/scripts/test-guardoni.sh b/scripts/test-guardoni.sh new file mode 100755 index 000000000..7eeb26b13 --- /dev/null +++ b/scripts/test-guardoni.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# set -e -x +set -e + +docker-compose up -d mongo-tk-test-indexes mongo-yt-test-indexes + +cd ./platforms/guardoni || exit; + +./scripts/cli-build.mjs +./scripts/cli-yt-test-home.mjs +./scripts/cli-yt-test-videos.mjs + +yarn pm2 stop all diff --git a/scripts/test-tk.sh b/scripts/test-tk.sh deleted file mode 100755 index 522937760..000000000 --- a/scripts/test-tk.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# set -e -x -set -e - - -# TK -yarn pm2 start ./platforms/tktrex/backend/ecosystem.config.js - -# register an experiment for home -search_experiment_register_out="$(yarn guardoni cli --verbose --basePath ./ -c guardoni.config.json tk-register ./experiments/tk-search.csv | grep 'experimentId:')" -search_experiment_id=${search_experiment_register_out/'experimentId: '/''} -echo $search_experiment_id - -# # exec the experiment -search_experiment_run_out=$(yarn guardoni cli --verbose --basePath ./ tk-experiment $search_experiment_id | grep 'publicKey:') -search_experiment_public_key=${search_experiment_run_out/'publicKey: '/''} -echo $search_experiment_public_key -echo "http://localhost:9000/api/v1/personal/$search_experiment_public_key" - -# curl "http://localhost:9000/api/v1/personal/$home_experiment_public_key" - - -# # register an experiment for videos -# video_experiment_register_out="$(yarn guardoni cli --verbose --backend http://localhost:9000/api yt-register ./experiments/tk-videos.csv | grep 'experimentId:')" -# video_experiment_id=${video_experiment_register_out/'experimentId: '/''} - -# echo $video_experiment_id - -# # exec the experiment -# video_experiment_run_out=$(yarn guardoni cli --verbose --backend http://localhost:9000/api yt-experiment -c guardoni.config.json $video_experiment_id | grep 'publicKey:') -# video_experiment_public_key=${video_experiment_run_out/'publicKey: '/''} -# echo $video_experiment_public_key - -# echo "Sleeping 30 seconds to let the parser process the data" -# # otherwise I was getting only the first 'home' as processed -# sleep 30; -# # curl "http://localhost:9000/api/v1/personal/$video_experiment_public_key" - - -yarn pm2 stop tk-trex