diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +node_modules diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..60d3b2f --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +15 diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..38ee235 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,13 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Run Script Using Node", + "type": "process", + "command": "node", + "args": ["${file}"] + } + ] +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f692605 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM livingdocs/node:15 +ADD package*.json /app/ +ENV NODE_ENV=production +RUN npm ci +ADD ./ /app +CMD ["node", "index.js"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..de52ea0 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# Loki log export to s3 buckets + +```bash +export AWS_REGION=eu-central-1 +export AWS_BUCKET=your-log-bucket +export AWS_ACCESS_KEY_ID=... +export AWS_SECRET_ACCESS_KEY=... +export LOKI_HOST=http://localhost:3100 + +export EXTRACTORS='[{ + "prefix": "some-service/", + "query": "{service=\"some-service\"}", + "transform": "json" +}, +{ + "prefix": "another-service/", + "query": "{service=\"another-service\"}", + "transform": "json" +}]' + +docker run --name loki-log-export livingdocs/loki-log-export:1.0.0 +``` + +Will run the log export every hour at 5 past and upload gzipped log files to s3: +``` +some-service/2020/12/01/00.log.gz +some-service/2020/12/01/01.log.gz +... +some-service/2020/12/01/23.log.gz +some-service/2020/12/02/00.log.gz + +another-service/2020/12/01/00.log.gz +another-service/2020/12/01/01.log.gz +... +another-service/2020/12/01/23.log.gz +another-service/2020/12/02/00.log.gz +``` diff --git a/exporter.js b/exporter.js new file mode 100644 index 0000000..2e2d2d3 --- /dev/null +++ b/exporter.js @@ -0,0 +1,155 @@ +const assert = require('assert') +const {promises: {pipeline}, Transform} = require('stream') +const CronJob = require('cron').CronJob +const zlib = require('zlib') +const getLogs = require('./get-logs') +const s3WriteStream = require('s3-streams').WriteStream +const S3 = require('aws-sdk').S3 + +module.exports = function createExporter (opts) { + const pino = require('pino')({base: null}) + + const {awsBucket, awsRegion, awsAccessKeyId, awsSecretAccessKey, lokiHost} = opts + assert(awsBucket, `The parameter 'opts.awsBucket' is required.`) + assert(awsRegion, `The parameter 'opts.awsRegion' is required.`) + assert(awsAccessKeyId, `The parameter 'opts.awsAccessKeyId' is required.`) + assert(awsSecretAccessKey, `The parameter 'opts.awsSecretAccessKey' is required.`) + assert(lokiHost, `The parameter 'opts.lokiHost' is required.`) + + const s3 = new S3({ + region: awsRegion, + accessKeyId: awsAccessKeyId, + secretAccessKey: awsSecretAccessKey + }) + + const createS3WriteStream = (key) => new s3WriteStream(s3, {Bucket: awsBucket, Key: key}) + + function toDateKey (prefix, date) { + return function (hour) { + const y = date.getFullYear() + const m = `${date.getMonth() + 1}`.padStart(2, 0) + const d = `${date.getDate()}`.padStart(2, 0) + const h = `${hour}`.padStart(2, 0) + const start = new Date(date) + start.setHours(hour, 0, 0, 0) + + const end = new Date(date) + end.setHours(hour + 1, 0, 0, 0) + + return { + key: `${y}-${m}-${d}/${h}.log.gz`.replace(/^\/?/, prefix || ''), + start, + end + } + } + } + + const hoursOfDay = Object.freeze([ + 0, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23 + ]) + + async function getHoursToProcess (prefix) { + const days = [] + const today = new Date() + days.push(hoursOfDay.slice(0, today.getHours()).map(toDateKey(prefix, today))) + + // Preload x days + for (let i = 1; i < 2; i++) { + const pastDay = new Date() + pastDay.setDate(today.getDate() - i) + pastDay.setHours(0,0,0,0) + days.push(hoursOfDay.map(toDateKey(prefix, pastDay))) + } + + const keys = [] + for (const day of days) { + if (!day.length) continue + const {Contents} = await s3.listObjectsV2({ + Bucket: awsBucket, + Prefix: day[0].key.replace(/..\.log\.gz$/, '') + }).promise() + keys.push(...Contents.map(({Key}) => Key)) + } + + const existing = new Set(keys) + const toProcess = [] + for (const day of days) { + for (const hour of day) { + if (!existing.has(hour.key)) toProcess.push(hour) + } + } + + // Returns an array of objects with keys + // {start: date, end: date, key: 'prefix/2020/11/01/00.log.gz'} + // {start: date, end: date, key: 'prefix/2020/11/01/...log.gz'} + // {start: date, end: date, key: 'prefix/2020/11/01/23.log.gz'} + return toProcess + } + + async function start (extractor) { + assert(extractor.query, `The parameter 'extractor.query' is required.`) + assert(extractor.transform, `The parameter 'extractor.transform' is required.`) + if (extractor.transform === 'json') { + extractor.transform = jsonTransform + } else if (typeof extractor.transform === 'string') { + extractor.transform = (new Function(`return ${extractor.transform}`))() + } + + const hours = await getHoursToProcess(extractor.prefix) + for (const hour of hours) { + const now = Date.now() + pino.info(`Processing logs for ${hour.key}`) + await pipeline( + getLogs({ + log: pino, + baseURL: lokiHost, + query: extractor.query, + start: hour.start, + end: hour.end + }), + logsToText(extractor), + zlib.createGzip(), + createS3WriteStream(hour.key) + ) + pino.info(`Persisted logs for ${hour.key}. Took ${Date.now() - now}ms.`) + } + } + + function startCron (extractor) { + const job = new CronJob('05 * * * *', () => start(extractor)) + return job.start() + } + + function jsonTransform ({value}) { + // Fix varnish user agents + value = value.replace(/\\x[a-f0-9]{2}/g, '') + JSON.parse(value) + return value + } + + function logsToText ({transform}) { + return new Transform({ + objectMode: true, + transform (lines, _, done) { + let str = '' + for (const line of lines) { + try { + const log = transform(line) + if (log) str += `${log}\n` + } catch (err) { + pino.info({err}, `Parsing of line failed: ${line.value}`) + } + } + done(null, str) + } + }) + } + + return { + log: pino, + start, + startCron + } +} diff --git a/get-logs.js b/get-logs.js new file mode 100644 index 0000000..197eea5 --- /dev/null +++ b/get-logs.js @@ -0,0 +1,53 @@ +const mergeStreams = require('./merge-logs') +const delay = require('util').promisify(setTimeout) + +module.exports = function follow (opts) { + const loki = require('axios').create({baseURL: opts.baseURL || 'http://localhost:3100'}) + const afterMs = opts.start.getTime ? opts.start.getTime() : Date.parse(opts.start) + const endMs = opts.end.getTime ? opts.end.getTime() : Date.parse(opts.end) + const query = opts.query || '{service="bluewin-test/varnish"}' + + return { + [Symbol.asyncIterator]() { + let after = `${afterMs}000000` + const end = `${endMs}000000` + return { + async next() { + let tries = 3 + while (tries--) { + try { + opts.log.debug(`Fetch after ${after}, end ${end}`) + const res = await loki({ + url: '/loki/api/v1/query_range', + params: { + limit: 5000, + direction: 'forward', + query, + start: after, + end + } + }) + + if (res.data.status !== 'success') { + throw new Error(`Invalid Loki Result: ${JSON.stringify(res.data)}`) + } + + const logs = mergeStreams(res.data.data.result) + + if (logs && logs[0] && logs[0].ts === after) logs.shift() + if (!logs.length) return {done: true} + + after = logs[logs.length - 1].ts + + return {done: false, value: logs} + } catch (err) { + opts.log.error({err}, 'Failed to fetch logs from loki') + } + await delay(1000) + } + throw new Error(`Failed to fetch logs after 3 retries.`) + } + } + } + } +} diff --git a/index.js b/index.js new file mode 100644 index 0000000..1846810 --- /dev/null +++ b/index.js @@ -0,0 +1,32 @@ +const extractors = JSON.parse(process.env.EXTRACTORS || '[]') +const exporter = require('./exporter')({ + awsBucket: process.env.AWS_BUCKET, + awsRegion: process.env.AWS_REGION, + awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID, + awsSecretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + lokiHost: process.env.LOKI_HOST || 'http://localhost:3100' +}) + +for (const extractor of extractors) { + exporter.log.warn(`Processor started for ${extractor.prefix}: ${extractor.query}`) + if (process.argv.includes('--once')) { + exporter.start(extractor) + } else { + exporter.startCron(extractor) + } +} + +const prexit = require('prexit') +prexit.signals.push('uncaughtException', 'unhandledRejection') +prexit.logExceptions = false + +prexit(async (signal, error) => { + const uptime = Math.round(process.uptime() * 100) / 100 + if ([0, 'SIGTERM', 'SIGINT'].includes(signal)) { + if (signal === 0) exporter.log.warn(`Shutting down after running for ${uptime}s`) + else exporter.log.warn(`Signal ${signal} received. Shutting down after running for ${uptime}s`) + } else { + const err = signal instanceof Error ? signal : error + exporter.log.fatal({err}, `Processing error. Shutting down after running for ${uptime}s`) + } +}) diff --git a/merge-logs.js b/merge-logs.js new file mode 100644 index 0000000..45ef65d --- /dev/null +++ b/merge-logs.js @@ -0,0 +1,38 @@ +module.exports = mergeSort + +// Merges multiple loki streams into one single array of logs +// transforms: +// [ +// {stream: {container_id: '123'}, values: [["1606521574140524000", "first"]]} +// {stream: {container_id: '321'}, values: [["1606521574140524001", "second"]]} +// ] +// to a sorted array by ts: +// [ +// {ts: "1606521574140524000", value: "first", stream: {container_id: '123'}} +// {ts: "1606521574140524001", value: "second", stream: {container_id: '321'}} +// ] +function mergeSort (streams, transform = (entry) => entry[1]) { + if (!streams.length) return [] + + const result = [] + + let hasEntries = true + while (hasEntries) { + let lowest = 0 + for (let i = 0; i < streams.length; i++) { + if (streams[i].values[0]?.[0] < (streams[lowest].values[0]?.[0] || Infinity)) lowest = i + } + + if (!streams[lowest]) throw new Error(`Fatal error: ${JSON.stringify(streams)}`) + + const elem = streams[lowest].values.shift() + if (!elem) hasEntries = false + else result.push({ts: elem[0], value: elem[1], stream: streams[lowest].stream}) + } + + return result +} + +// const data = require('./log.json') +// const result = mergeSort(data, ([ts, str], stream) => `${ts} - ${str}`) +// console.log(result[0], result[1], result[result.length - 1]) diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..7ddd5f0 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,543 @@ +{ + "name": "loki-log-export", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "aws-sdk": "^2.800.0", + "axios": "^0.21.0", + "cron": "^1.8.2", + "pino": "^6.7.0", + "prexit": "^0.0.5", + "s3-streams": "^0.4.0" + } + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/aws-sdk": { + "version": "2.800.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.800.0.tgz", + "integrity": "sha512-1vxT8h2iFFaxfn+onMIdL/L+OcIG6G4BUzRt4wuvQzwbKArRy0fUN2d6E7aoM74wl8VlNIMTxBLCqxDnfVeVxQ==", + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/axios": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz", + "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==", + "dependencies": { + "follow-redirects": "^1.10.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/cron": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/cron/-/cron-1.8.2.tgz", + "integrity": "sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg==", + "dependencies": { + "moment-timezone": "^0.5.x" + } + }, + "node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/fast-redact": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.0.0.tgz", + "integrity": "sha512-a/S/Hp6aoIjx7EmugtzLqXmcNsyFszqbt6qQ99BdG61QjBZF6shNis0BYR6TsZOQ1twYc0FN2Xdhwwbv6+KD0w==", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-safe-stringify": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", + "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" + }, + "node_modules/flatstr": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", + "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" + }, + "node_modules/follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "node_modules/moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", + "engines": { + "node": "*" + } + }, + "node_modules/moment-timezone": { + "version": "0.5.32", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.32.tgz", + "integrity": "sha512-Z8QNyuQHQAmWucp8Knmgei8YNo28aLjJq6Ma+jy1ZSpSk5nyfRT8xgUbSQvD2+2UajISfenndwvFuH3NGS+nvA==", + "dependencies": { + "moment": ">= 2.9.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/pino": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.7.0.tgz", + "integrity": "sha512-vPXJ4P9rWCwzlTJt+f0Ni4THc3DWyt8iDDCO4edQ8narTu6hnpzdXu8FqeSJCGndl1W6lfbYQUQihUO54y66Lw==", + "dependencies": { + "fast-redact": "^3.0.0", + "fast-safe-stringify": "^2.0.7", + "flatstr": "^1.0.12", + "pino-std-serializers": "^2.4.2", + "quick-format-unescaped": "^4.0.1", + "sonic-boom": "^1.0.2" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-std-serializers": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.5.0.tgz", + "integrity": "sha512-wXqbqSrIhE58TdrxxlfLwU9eDhrzppQDvGhBEr1gYbzzM4KKo3Y63gSjiDXRKLVS2UOXdPNR2v+KnQgNrs+xUg==" + }, + "node_modules/prexit": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/prexit/-/prexit-0.0.5.tgz", + "integrity": "sha512-S2dauVLdYACgrRnjn+Cr387T27w1ONZKYcEosDW4v7lWhlxDNK3cge7gNK7UfkD561X3rftyjrpBB4LcjZ+2Hw==" + }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz", + "integrity": "sha512-RyYpQ6Q5/drsJyOhrWHYMWTedvjTIat+FTwv0K4yoUxzvekw2aRHMQJLlnvt8UantkZg2++bEzD9EdxXqkWf4A==" + }, + "node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/s3-streams": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/s3-streams/-/s3-streams-0.4.0.tgz", + "integrity": "sha512-DtZ7w3A0EorzHdhh00U3p7O2c2hv2w/i+A1JATAJZubp+fnwlU8MiejJibAzMLhhCRv+UsfimSGoivWt2Y4JsQ==", + "dependencies": { + "bluebird": "^3.5.3", + "lodash": "^4.17.11", + "readable-stream": "^3.1.1" + }, + "engines": { + "iojs": "^1.2.0", + "node": "^0.8 || ^0.10 || ^0.12" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "node_modules/sonic-boom": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.3.0.tgz", + "integrity": "sha512-4nX6OYvOYr6R76xfQKi6cZpTO3YSWe/vd+QdIfoH0lBy0MnPkeAbb2rRWgmgADkXUeCKPwO1FZAKlAVWAadELw==", + "dependencies": { + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/url/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "node_modules/uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "node_modules/xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "engines": { + "node": ">=4.0" + } + } + }, + "dependencies": { + "atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" + }, + "aws-sdk": { + "version": "2.800.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.800.0.tgz", + "integrity": "sha512-1vxT8h2iFFaxfn+onMIdL/L+OcIG6G4BUzRt4wuvQzwbKArRy0fUN2d6E7aoM74wl8VlNIMTxBLCqxDnfVeVxQ==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, + "axios": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz", + "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "cron": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/cron/-/cron-1.8.2.tgz", + "integrity": "sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg==", + "requires": { + "moment-timezone": "^0.5.x" + } + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "fast-redact": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.0.0.tgz", + "integrity": "sha512-a/S/Hp6aoIjx7EmugtzLqXmcNsyFszqbt6qQ99BdG61QjBZF6shNis0BYR6TsZOQ1twYc0FN2Xdhwwbv6+KD0w==" + }, + "fast-safe-stringify": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", + "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" + }, + "flatstr": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", + "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" + }, + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, + "lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" + }, + "moment-timezone": { + "version": "0.5.32", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.32.tgz", + "integrity": "sha512-Z8QNyuQHQAmWucp8Knmgei8YNo28aLjJq6Ma+jy1ZSpSk5nyfRT8xgUbSQvD2+2UajISfenndwvFuH3NGS+nvA==", + "requires": { + "moment": ">= 2.9.0" + } + }, + "pino": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.7.0.tgz", + "integrity": "sha512-vPXJ4P9rWCwzlTJt+f0Ni4THc3DWyt8iDDCO4edQ8narTu6hnpzdXu8FqeSJCGndl1W6lfbYQUQihUO54y66Lw==", + "requires": { + "fast-redact": "^3.0.0", + "fast-safe-stringify": "^2.0.7", + "flatstr": "^1.0.12", + "pino-std-serializers": "^2.4.2", + "quick-format-unescaped": "^4.0.1", + "sonic-boom": "^1.0.2" + } + }, + "pino-std-serializers": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.5.0.tgz", + "integrity": "sha512-wXqbqSrIhE58TdrxxlfLwU9eDhrzppQDvGhBEr1gYbzzM4KKo3Y63gSjiDXRKLVS2UOXdPNR2v+KnQgNrs+xUg==" + }, + "prexit": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/prexit/-/prexit-0.0.5.tgz", + "integrity": "sha512-S2dauVLdYACgrRnjn+Cr387T27w1ONZKYcEosDW4v7lWhlxDNK3cge7gNK7UfkD561X3rftyjrpBB4LcjZ+2Hw==" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "quick-format-unescaped": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz", + "integrity": "sha512-RyYpQ6Q5/drsJyOhrWHYMWTedvjTIat+FTwv0K4yoUxzvekw2aRHMQJLlnvt8UantkZg2++bEzD9EdxXqkWf4A==" + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "s3-streams": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/s3-streams/-/s3-streams-0.4.0.tgz", + "integrity": "sha512-DtZ7w3A0EorzHdhh00U3p7O2c2hv2w/i+A1JATAJZubp+fnwlU8MiejJibAzMLhhCRv+UsfimSGoivWt2Y4JsQ==", + "requires": { + "bluebird": "^3.5.3", + "lodash": "^4.17.11", + "readable-stream": "^3.1.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "sonic-boom": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.3.0.tgz", + "integrity": "sha512-4nX6OYvOYr6R76xfQKi6cZpTO3YSWe/vd+QdIfoH0lBy0MnPkeAbb2rRWgmgADkXUeCKPwO1FZAKlAVWAadELw==", + "requires": { + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..cbd695d --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "loki-log-export", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "aws-sdk": "^2.800.0", + "axios": "^0.21.0", + "cron": "^1.8.2", + "pino": "^6.7.0", + "prexit": "^0.0.5", + "s3-streams": "^0.4.0" + } +} diff --git a/test/example-loki-log.json b/test/example-loki-log.json new file mode 100644 index 0000000..5551c68 --- /dev/null +++ b/test/example-loki-log.json @@ -0,0 +1,82 @@ +[ + { + "stream": { + "container_id": "5f5fe11219aa2731b09f4a30ffab166fb6b03f2ea69bd69a61ad92927ee6e7e8", + "container_image": "livingdocs/varnish:6.4.0-r3", + "container_name": "r-bluewin-test-varnish-15-1138dbcf", + "host": "worker-dev-giv4lc", + "service": "bluewin-test/varnish", + "stream": "journal", + "unit": "docker.service", + "cluster": "development" + }, + "values": [ + [ + "1606435205397852000", + "{\"@timestamp\":\"2020-11-27T00:00:03+0000\",\"method\":\"GET\",\"url\":\"/\",\"remote_ip\":\"10.42.51.81\",\"x_forwarded_for\":\"10.42.3.228, 10.42.51.81\",\"cache\":\"miss\",\"bytes\":\"80\",\"duration_usec\":\"2964\",\"status\":\"302\",\"request\":\"GET http://bluewin.livingdocs.io/ HTTP/2.0\",\"ttfb\":\"0.002709\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435213854322000", + "{\"@timestamp\":\"2020-11-27T00:00:13+0000\",\"method\":\"GET\",\"url\":\"/de/index.html\",\"remote_ip\":\"10.42.51.81\",\"x_forwarded_for\":\"10.42.3.228, 10.42.51.81\",\"cache\":\"hit\",\"bytes\":\"108039\",\"duration_usec\":\"521\",\"status\":\"200\",\"request\":\"GET http://bluewin.livingdocs.io/de/index.html HTTP/2.0\",\"ttfb\":\"0.000190\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435235221104000", + "{\"@timestamp\":\"2020-11-27T00:00:33+0000\",\"method\":\"GET\",\"url\":\"/\",\"remote_ip\":\"10.42.222.6\",\"x_forwarded_for\":\"161.35.66.180, 10.42.222.6\",\"cache\":\"miss\",\"bytes\":\"80\",\"duration_usec\":\"2129\",\"status\":\"302\",\"request\":\"GET http://bluewin.livingdocs.io/ HTTP/2.0\",\"ttfb\":\"0.002054\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435243859975000", + "{\"@timestamp\":\"2020-11-27T00:00:43+0000\",\"method\":\"GET\",\"url\":\"/de/index.html\",\"remote_ip\":\"10.42.222.6\",\"x_forwarded_for\":\"161.35.66.180, 10.42.222.6\",\"cache\":\"hit\",\"bytes\":\"108039\",\"duration_usec\":\"294\",\"status\":\"200\",\"request\":\"GET http://bluewin.livingdocs.io/de/index.html HTTP/2.0\",\"ttfb\":\"0.000143\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435275399180000", + "{\"@timestamp\":\"2020-11-27T00:01:13+0000\",\"method\":\"GET\",\"url\":\"/\",\"remote_ip\":\"10.42.51.81\",\"x_forwarded_for\":\"10.42.3.228, 10.42.51.81\",\"cache\":\"miss\",\"bytes\":\"80\",\"duration_usec\":\"2292\",\"status\":\"302\",\"request\":\"GET http://bluewin.livingdocs.io/ HTTP/2.0\",\"ttfb\":\"0.002101\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ] + ] + }, + { + "stream": { + "host": "worker-dev-gmlq89", + "service": "bluewin-test/varnish", + "stream": "journal", + "unit": "docker.service", + "cluster": "development", + "container_id": "9e6df53d38c855228b3215e5d1ec053f96dab5dac289efd8f733560afcb7eddf", + "container_image": "livingdocs/varnish:6.4.0-r3", + "container_name": "r-bluewin-test-varnish-13-54ba2709" + }, + "values": [ + [ + "1606435203856851000", + "{\"@timestamp\":\"2020-11-27T00:00:03+0000\",\"method\":\"GET\",\"url\":\"/de/index.html\",\"remote_ip\":\"10.42.51.81\",\"x_forwarded_for\":\"10.42.3.228, 10.42.51.81\",\"cache\":\"hit\",\"bytes\":\"108039\",\"duration_usec\":\"500\",\"status\":\"200\",\"request\":\"GET http://bluewin.livingdocs.io/de/index.html HTTP/2.0\",\"ttfb\":\"0.000180\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435225219558000", + "{\"@timestamp\":\"2020-11-27T00:00:23+0000\",\"method\":\"GET\",\"url\":\"/\",\"remote_ip\":\"10.42.222.6\",\"x_forwarded_for\":\"161.35.66.180, 10.42.222.6\",\"cache\":\"miss\",\"bytes\":\"80\",\"duration_usec\":\"1508\",\"status\":\"302\",\"request\":\"GET http://bluewin.livingdocs.io/ HTTP/2.0\",\"ttfb\":\"0.001274\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435233875061000", + "{\"@timestamp\":\"2020-11-27T00:00:33+0000\",\"method\":\"GET\",\"url\":\"/de/index.html\",\"remote_ip\":\"10.42.222.6\",\"x_forwarded_for\":\"161.35.66.180, 10.42.222.6\",\"cache\":\"hit\",\"bytes\":\"108039\",\"duration_usec\":\"452\",\"status\":\"200\",\"request\":\"GET http://bluewin.livingdocs.io/de/index.html HTTP/2.0\",\"ttfb\":\"0.000156\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435255468985000", + "{\"@timestamp\":\"2020-11-27T00:00:53+0000\",\"method\":\"GET\",\"url\":\"/\",\"remote_ip\":\"10.42.138.233\",\"x_forwarded_for\":\"161.35.66.180, 10.42.138.233\",\"cache\":\"miss\",\"bytes\":\"80\",\"duration_usec\":\"1568\",\"status\":\"302\",\"request\":\"GET http://bluewin.livingdocs.io/ HTTP/2.0\",\"ttfb\":\"0.001481\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435265396698000", + "{\"@timestamp\":\"2020-11-27T00:01:03+0000\",\"method\":\"GET\",\"url\":\"/\",\"remote_ip\":\"10.42.51.81\",\"x_forwarded_for\":\"10.42.3.228, 10.42.51.81\",\"cache\":\"miss\",\"bytes\":\"80\",\"duration_usec\":\"1922\",\"status\":\"302\",\"request\":\"GET http://bluewin.livingdocs.io/ HTTP/2.0\",\"ttfb\":\"0.001708\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435273845734000", + "{\"@timestamp\":\"2020-11-27T00:01:13+0000\",\"method\":\"GET\",\"url\":\"/de/index.html\",\"remote_ip\":\"10.42.51.81\",\"x_forwarded_for\":\"10.42.3.228, 10.42.51.81\",\"cache\":\"hit\",\"bytes\":\"108039\",\"duration_usec\":\"400\",\"status\":\"200\",\"request\":\"GET http://bluewin.livingdocs.io/de/index.html HTTP/2.0\",\"ttfb\":\"0.000156\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435283870424000", + "{\"@timestamp\":\"2020-11-27T00:01:23+0000\",\"method\":\"GET\",\"url\":\"/de/index.html\",\"remote_ip\":\"10.42.138.233\",\"x_forwarded_for\":\"161.35.66.180, 10.42.138.233\",\"cache\":\"hit\",\"bytes\":\"108039\",\"duration_usec\":\"255\",\"status\":\"200\",\"request\":\"GET http://bluewin.livingdocs.io/de/index.html HTTP/2.0\",\"ttfb\":\"0.000117\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ], + [ + "1606435293852023000", + "{\"@timestamp\":\"2020-11-27T00:01:33+0000\",\"method\":\"GET\",\"url\":\"/de/index.html\",\"remote_ip\":\"10.42.51.81\",\"x_forwarded_for\":\"10.42.3.228, 10.42.51.81\",\"cache\":\"hit\",\"bytes\":\"108039\",\"duration_usec\":\"379\",\"status\":\"200\",\"request\":\"GET http://bluewin.livingdocs.io/de/index.html HTTP/2.0\",\"ttfb\":\"0.000126\",\"referrer\":\"-\",\"user_agent\":\"Go-http-client/1.1\"}" + ] + ] + } +]