diff --git a/package.json b/package.json index 0c12a01..870094b 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ }, "license": "MIT", "dependencies": { + "@types/lokijs": "^1.5.2", "@types/mkdirp": "^0.5.2", "content-type": "^1.0.4", "debug": "^4.1.1", @@ -78,6 +79,7 @@ "incoming-message-hash": "^3.2.2", "invariant": "^2.2.4", "lodash.map": "^4.6.0", + "lokijs": "^1.5.6", "mkdirp": "^0.5.1", "restify-errors": "^6.1.1", "xxhashjs": "^0.2.2" diff --git a/src/Recorder.ts b/src/Recorder.ts new file mode 100644 index 0000000..a406048 --- /dev/null +++ b/src/Recorder.ts @@ -0,0 +1,155 @@ +import { YakTimeOpts, ensureRecordingIsAllowed, ensureIsValidStatusCode } from './util' +import Loki from 'lokijs' +import { getDB } from './db' +import Debug from 'debug' +import { IncomingMessage, ServerResponse, IncomingHttpHeaders } from 'http' +import { URL } from 'url' +import { h64 } from 'xxhashjs' +import { parse } from 'querystring' +import { buffer } from './buffer' +import { proxy } from './proxy' +import * as curl from './curl' +import { isMatch } from 'lodash' + +const debug = Debug('yaktime:recorder') + +type Unpacked = T extends (infer U)[] ? U : T extends (...args: any[]) => infer U ? U : T extends Promise ? U : T + +type SerializedRequest = ReturnType +type SerializedResponse = Unpacked> +interface FullSerializedRequest extends SerializedRequest { + response: SerializedResponse +} + +export class Recorder { + opts: YakTimeOpts + host: string + db: Promise + constructor (opts: YakTimeOpts, host: string) { + this.opts = opts + this.host = host + this.db = getDB(opts) + } + + serializeRequest (req: IncomingMessage, body: any[]) { + const fullUrl = new URL(req.url as string, this.host) + const { method = '', httpVersion, headers, trailers } = req + + const bodyBuffer = Buffer.from(body) + const bodyEncoded = bodyBuffer.toString('base64') + const bodyHash = h64(bodyBuffer, 0).toString(16) + + return { + host: fullUrl.host, + path: fullUrl.pathname, + body: bodyEncoded, + bodyHash, + method, + httpVersion, + headers, + trailers, + query: parse(fullUrl.searchParams.toString()) + } + } + + async serializeResponse (res: IncomingMessage) { + const statusCode = res.statusCode || 200 + const headers = res.headers + const body = Buffer.from(await buffer(res)).toString('base64') + const trailers = res.trailers + + return { + statusCode, + headers, + body, + trailers + } + } + + async respond (storedRes: SerializedResponse, res: ServerResponse) { + res.statusCode = storedRes.statusCode + res.writeHead(storedRes.statusCode, storedRes.headers) + res.addTrailers(storedRes.trailers || {}) + console.log(storedRes.body) + res.end(Buffer.from(storedRes.body, 'base64')) + } + + async record (req: IncomingMessage, body: Buffer[], host: string, opts: YakTimeOpts) { + ensureRecordingIsAllowed(req, opts) + debug('proxy', req.url) + const pres = await proxy(req, body, host) + debug(curl.response(pres)) + ensureIsValidStatusCode(pres, opts) + debug('record', req.url) + const request = this.serializeRequest(req, body) + const response = await this.serializeResponse(pres) + return this.save(request, response) + } + + async save (request: SerializedRequest, response: SerializedResponse) { + const db = await this.db + const tapes = db.addCollection('tapes', { disableMeta: true }) + return tapes.add({ ...request, response }) + } + + async read (req: IncomingMessage, body: Buffer[]) { + const serializedRequest = this.serializeRequest(req, body) + return this.load(serializedRequest) + } + + async load (request: SerializedRequest): Promise { + const { ignoredQueryFields = [], ignoredHeaders = [] } = this.opts.hasherOptions || {} + const db = await this.db + const tapes = db.addCollection('tapes', { disableMeta: true }) + + const { query: _query, headers: _headers } = request + + const query = { + ..._query + } + const headers = { + ..._headers + } + + ignoredQueryFields.forEach(q => delete query[q]) + ignoredHeaders.forEach(h => delete headers[h]) + + const lokiQuery = { + ...request, + query, + headers + } + + delete query.body + + return tapes.where(obj => isMatch(obj, lokiQuery))[0] + } +} + +export class DbMigrator { + data: Buffer[] = [] + headers: IncomingHttpHeaders = {} + statusCode = 200 + setHeader (name: string, value: string) { + this.headers[name] = value + } + write (input: Buffer | string) { + this.data.push(Buffer.isBuffer(input) ? input : Buffer.from(input)) + } + + end (data?: any) { + if (data != null) { + this.write(data) + } + debug('finished migration') + } + + toSerializedResponse (): SerializedResponse { + return { + statusCode: this.statusCode, + headers: this.headers, + body: Buffer.concat(this.data).toString('base64'), + trailers: {} + } + } +} diff --git a/src/db.ts b/src/db.ts new file mode 100644 index 0000000..db105b5 --- /dev/null +++ b/src/db.ts @@ -0,0 +1,15 @@ +import Loki from 'lokijs' +import { YakTimeOpts } from './util' +import path from 'path' +import Debug from 'debug' + +const debug = Debug('yaktime:db') +let db: Loki +export async function getDB (opts: YakTimeOpts) { + if (db == null) { + const dbPath = path.join(opts.dirname, 'tapes.json') + debug(`Opening db on ${dbPath}`) + db = new Loki(dbPath, { autoload: true, autosave: process.env.NODE_ENV !== 'test' }) + } + return db +} diff --git a/src/proxy.ts b/src/proxy.ts index 6d5dbf8..ef1adf4 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -16,7 +16,7 @@ const debug = Debug('yaktime:proxy') const mods = { 'http:': http, 'https:': https } -interface YakTimeIncomingMessage extends http.IncomingMessage { +export interface YakTimeIncomingMessage extends http.IncomingMessage { req: http.ClientRequest } diff --git a/src/record.ts b/src/record.ts index cde893d..749c526 100644 --- a/src/record.ts +++ b/src/record.ts @@ -10,7 +10,7 @@ import { ModuleNotFoundError } from './errors' import { proxy } from './proxy' import * as curl from './curl' -const debug = Debug('yaktime:record') +const debug = Debug('yaktime:legacy-record') /** * Write `data` to `filename`. diff --git a/src/tapeMigrator.test.ts b/src/tapeMigrator.test.ts index 02d88cb..9665b2f 100644 --- a/src/tapeMigrator.test.ts +++ b/src/tapeMigrator.test.ts @@ -9,11 +9,12 @@ import * as path from 'path' import { createServer, TestServer } from './test/helpers/server' import { createTmpdir, Dir } from './test/helpers/tmpdir' import { AddressInfo } from 'net' -import { tapename, RequestHasher } from './util' +import { tapename, RequestHasher, YakTimeOpts } from './util' import { requestHasher } from './requestHasher' -import { tapeMigrator } from './tapeMigrator' +import { fileTapeMigrator, dbTapeMigrator } from './tapeMigrator' import { yaktime } from './yaktime' import { notifyNotUsedTapes } from './tracker' +import { Recorder } from './Recorder' const incMessH = require('incoming-message-hash') const messageHash: RequestHasher = incMessH.sync @@ -57,14 +58,14 @@ describe('record', () => { test('copies the file with the new name', done => { expect.hasAssertions() - req.once('response', async function() { + req.once('response', async function () { const serverReq = proxyServer.requests[0] const newHash = requestHasher({}) const oldFileName = path.join(tmpdir.dirname, tapename(messageHash, serverReq)) const newFileName = path.join(tmpdir.dirname, tapename(newHash, serverReq)) expect(fs.existsSync(oldFileName)).toEqual(true) expect(fs.existsSync(newFileName)).toEqual(false) - await tapeMigrator(newHash, { dirname: tmpdir.dirname })(serverReq) + await fileTapeMigrator(newHash, { dirname: tmpdir.dirname })(serverReq) expect(fs.existsSync(oldFileName)).toEqual(true) expect(fs.existsSync(newFileName)).toEqual(true) done() @@ -72,4 +73,31 @@ describe('record', () => { req.end() }) + + test('add existing request to the db', done => { + expect.hasAssertions() + + req.once('response', async function () { + const migrator = new Recorder({ dirname: tmpdir.dirname, useDb: true } as YakTimeOpts, `http://localhost:${serverInfo.port}`) + const serverReq = proxyServer.requests[0] + const oldFileName = path.join(tmpdir.dirname, tapename(messageHash, serverReq)) + expect(fs.existsSync(oldFileName)).toEqual(true) + expect(await migrator.read(serverReq, [])).toBeUndefined() + await dbTapeMigrator(`http://localhost:${serverInfo.port}`, { dirname: tmpdir.dirname, useDb: true })(serverReq) + expect(fs.existsSync(oldFileName)).toEqual(true) + expect(await migrator.read(serverReq, [])).toEqual( + expect.objectContaining({ + method: 'GET', + path: '/', + response: expect.objectContaining({ + statusCode: 201, + body: 'T0s=' + }) + }) + ) + done() + }) + + req.end() + }) }) diff --git a/src/tapeMigrator.ts b/src/tapeMigrator.ts index 7ab3aa5..caea9a6 100644 --- a/src/tapeMigrator.ts +++ b/src/tapeMigrator.ts @@ -4,6 +4,7 @@ import Debug from 'debug' import { IncomingMessage } from 'http' import { existsSync } from 'fs' import { requestHasher } from './requestHasher' +import { Recorder, DbMigrator } from './Recorder' const debug = Debug('yaktime:tape-migrator') @@ -11,14 +12,14 @@ const incMessH = require('incoming-message-hash') const oldHasher: RequestHasher = incMessH.sync type tapeMigratorOptions = 'dirname' | 'oldHash' -export const tapeMigrator = (newHasher: RequestHasher, opts: Pick) => +export const fileTapeMigrator = (newHasher: RequestHasher, opts: Pick) => async function tapeMigrator (req: IncomingMessage, body: Buffer[] = []) { const oldFileName = path.join(opts.dirname, tapename(opts.oldHash || oldHasher, req, body)) const newFileName = path.join(opts.dirname, tapename(newHasher, req, body)) const oldExists = existsSync(oldFileName) if (oldExists) { - debug('migrating') + debug('migrating to file') debug('old filename', oldFileName) const newExists = existsSync(newFileName) if (newExists) { @@ -31,8 +32,28 @@ export const tapeMigrator = (newHasher: RequestHasher, opts: Pick + async function tapeMigrator (req: IncomingMessage, body: Buffer[] = []) { + const recorder = new Recorder(opts, host) + const oldFileName = path.join(opts.dirname, tapename(opts.oldHash || oldHasher, req, body)) + const oldExists = existsSync(oldFileName) + + if (oldExists) { + debug('migrating to db') + debug('filename', oldFileName) + + const migrator = new DbMigrator() + require(oldFileName)(null, migrator) + const request = recorder.serializeRequest(req, body) + const response = migrator.toSerializedResponse() + debug('saving to db') + await recorder.save(request, response) + } + } + +type migrateIfRequiredOptions = 'hash' | 'migrate' | 'hasherOptions' | 'useDb' export async function migrateIfRequired ( + host: string, opts: Pick, req: IncomingMessage, body: Buffer[] = [] @@ -41,5 +62,6 @@ export async function migrateIfRequired ( return } const newHasher = requestHasher(opts.hasherOptions) - await tapeMigrator(newHasher, opts)(req, body) + await dbTapeMigrator(host, opts)(req, body) + await fileTapeMigrator(newHasher, opts)(req, body) } diff --git a/src/util.ts b/src/util.ts index 5f5ff54..5bb1f48 100644 --- a/src/util.ts +++ b/src/util.ts @@ -61,6 +61,11 @@ export interface YakTimeOpts { * this are only used when `opts.hash` is null */ hasherOptions?: RequestHasherOptions + /** + * Whether to use a built-in database instead of JS files. + * To avoid issues with hashers + */ + useDb?: boolean } export interface YakTimeServer { diff --git a/src/yaktime.test.ts b/src/yaktime.test.ts index bf34002..d4c7f9e 100644 --- a/src/yaktime.test.ts +++ b/src/yaktime.test.ts @@ -1,3 +1,4 @@ +// tslint:disable:no-duplicate-string // Copyright 2016 Yahoo Inc. // Copyright 2019 Ale Figueroa // Licensed under the terms of the MIT license. Please see LICENSE file in the project root for terms. @@ -14,6 +15,7 @@ import { RequestHasher } from './util' const fixedUA = 'node-superagent/0.21.0' +// tslint:disable-next-line:no-big-function describe('yakbak', () => { let server: TestServer let serverInfo: AddressInfo @@ -51,7 +53,7 @@ describe('yakbak', () => { .expect('X-Yakbak-Tape', '1a574e91da6cf00ac18bc97abaed139e') .expect('Content-Type', 'text/html') .expect(201, 'OK') - .end(function(err) { + .end(function (err) { expect(server.requests.length).toEqual(1) done(err) }) @@ -65,7 +67,7 @@ describe('yakbak', () => { .expect('X-Yakbak-Tape', '3234ee470c8605a1837e08f218494326') .expect('Content-Type', 'text/html') .expect(201, 'OK') - .end(function(err) { + .end(function (err) { expect(fs.existsSync(tmpdir.join('3234ee470c8605a1837e08f218494326.js'))).toBeTruthy() done(err) }) @@ -74,7 +76,7 @@ describe('yakbak', () => { describe('when given a custom hashing function', () => { beforeEach(() => { // customHash creates a MD5 of the request, ignoring its querystring, headers, etc. - const customHash: RequestHasher = function customHash(req, body) { + const customHash: RequestHasher = function customHash (req, body) { let hash = crypto.createHash('md5') let parts = url.parse(req.url as string, true) @@ -97,7 +99,7 @@ describe('yakbak', () => { .expect('X-Yakbak-Tape', '3f142e515cb24d1af9e51e6869bf666f') .expect('Content-Type', 'text/html') .expect(201, 'OK') - .end(function(err) { + .end(function (err) { expect(fs.existsSync(tmpdir.join('3f142e515cb24d1af9e51e6869bf666f.js'))).toBeTruthy() done(err) }) @@ -125,7 +127,7 @@ describe('yakbak', () => { .get('/record/2') .set('user-agent', fixedUA) .set('host', 'localhost:3001') - .end(function(err) { + .end(function (err) { expect(server.requests.length).toEqual(0) done(err) }) @@ -136,7 +138,7 @@ describe('yakbak', () => { .get('/record/2') .set('user-agent', fixedUA) .set('host', 'localhost:3001') - .end(function(err) { + .end(function (err) { expect(!fs.existsSync(tmpdir.join('3234ee470c8605a1837e08f218494326.js'))).toBeTruthy() done(err) }) @@ -167,7 +169,7 @@ describe('yakbak', () => { .set('user-agent', fixedUA) .set('host', 'localhost:3001') .expect(404) - .end(function(err) { + .end(function (err) { expect(!fs.existsSync(tmpdir.join('3234ee470c8605a1837e08f218494326.js'))).toBeTruthy() done(err) }) @@ -177,8 +179,12 @@ describe('yakbak', () => { describe('playback', () => { let yakbak: any - beforeEach(() => { + let yakbakWithDb: any + beforeEach(async () => { + await tmpdir.teardown() + await tmpdir.setup() yakbak = subject(`http://localhost:${serverInfo.port}`, { dirname: tmpdir.dirname }) + yakbakWithDb = subject(`http://localhost:${serverInfo.port}`, { dirname: tmpdir.dirname, useDb: true, migrate: true }) }) beforeEach(done => { @@ -197,18 +203,22 @@ describe('yakbak', () => { fs.writeFile(tmpdir.join(file), tape, done) }) - test('does not make a request to the server', done => { - request(yakbak) + const test1 = (yak: any, done: any) => + request(yak) .get('/playback/1') .set('user-agent', fixedUA) .set('host', 'localhost:3001') .expect('X-Yakbak-Tape', '305c77b0a3ad7632e51c717408d8be0f') .expect('Content-Type', 'text/html') .expect(201, 'YAY') - .end(function(err) { + .end(function (err) { expect(server.requests.length).toEqual(0) done(err) }) + + test('does not make a request to the server', done => { + test1(yakbak, done) + test1(yakbakWithDb, done) }) }) }) diff --git a/src/yaktime.ts b/src/yaktime.ts index 1e33100..cd173cd 100644 --- a/src/yaktime.ts +++ b/src/yaktime.ts @@ -8,12 +8,13 @@ import * as path from 'path' import Debug from 'debug' import { buffer } from './buffer' import { recordIfNotFound } from './record' -import { mkdir, RequestHasher, YakTimeOpts, YakTimeServer, tapename, resolveModule } from './util' +import { mkdir, RequestHasher, YakTimeOpts, YakTimeServer, tapename, resolveModule as resolveCassete } from './util' import { HttpError } from 'restify-errors' import * as curl from './curl' import { migrateIfRequired } from './tapeMigrator' import { requestHasher } from './requestHasher' import { trackHit } from './tracker' +import { Recorder } from './Recorder' const debug = Debug('yaktime:server') @@ -26,6 +27,7 @@ const messageHash: RequestHasher = incMessH.sync * @param - opts */ +// tslint:disable-next-line:cognitive-complexity export function yaktime (host: string, opts: YakTimeOpts): YakTimeServer { invariant(opts.dirname != null && opts.dirname !== '', 'You must provide opts.dirname') @@ -41,14 +43,22 @@ export function yaktime (host: string, opts: YakTimeOpts): YakTimeServer { const file = path.join(opts.dirname, tapename(opts.hash || defaultHasher, req, body)) try { - await migrateIfRequired(opts, req, body) - const filename = await resolveModule(file).catch(recordIfNotFound(req, body, host, file, opts)) - trackHit(filename, hits) - const tape: YakTimeServer = require(filename) - tape(req, res) + await migrateIfRequired(host, opts, req, body) + + if (opts.useDb === true) { + const recorder = new Recorder(opts, host) + const response = (await recorder.read(req, body)) || (await recorder.record(req, body, host, opts)) + await recorder.respond(response.response, res) + } else { + const filename = await resolveCassete(file).catch(recordIfNotFound(req, body, host, file, opts)) + + trackHit(filename, hits) + const tape: YakTimeServer = require(filename) + tape(req, res) + } } catch (err) { res.statusCode = err instanceof HttpError ? err.statusCode : 500 - debug(err.message) + debug(err.stack) res.end(err.message) } } diff --git a/yarn.lock b/yarn.lock index aa2256b..116c562 100644 --- a/yarn.lock +++ b/yarn.lock @@ -805,6 +805,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.120.tgz#cf265d06f6c7a710db087ed07523ab8c1a24047b" integrity sha512-jQ21kQ120mo+IrDs1nFNVm/AsdFxIx2+vZ347DbogHJPd/JzKNMOqU6HCYin1W6v8l5R9XSO2/e9cxmn7HAnVw== +"@types/lokijs@^1.5.2": + version "1.5.2" + resolved "https://registry.yarnpkg.com/@types/lokijs/-/lokijs-1.5.2.tgz#ed228f080033ce1fb16eff4acde65cb9ae0f1bf2" + integrity sha512-ZF14v1P1Bjbw8VJRu+p4WS9V926CAOjWF4yq23QmSBWRPe0/GXlUKzSxjP1fi/xi8nrq6zr9ECo8Z/8KsRqroQ== + "@types/minimatch@*": version "3.0.3" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" @@ -4275,6 +4280,11 @@ log-update@^2.3.0: cli-cursor "^2.0.0" wrap-ansi "^3.0.1" +lokijs@^1.5.6: + version "1.5.6" + resolved "https://registry.yarnpkg.com/lokijs/-/lokijs-1.5.6.tgz#6de6b8c3ff7a972fd0104169f81e7ddc244c029f" + integrity sha512-xJoDXy8TASTjmXMKr4F8vvNUCu4dqlwY5gmn0g5BajGt1GM3goDCafNiGAh/sfrWgkfWu1J4OfsxWm8yrWweJA== + loose-envify@^1.0.0, loose-envify@^1.3.1: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"