From 7009abf3e39a9cf1a776bace4de3a8ad6f680157 Mon Sep 17 00:00:00 2001 From: joe-fol Date: Thu, 26 Sep 2024 09:10:44 +0100 Subject: [PATCH] Remove pnld download --- package-lock.json | 726 +----------------------- src/pnld-download/PnldFileDownloader.ts | 151 ----- src/pnld-download/config.ts | 35 -- src/pnld-download/convertXml.ts | 49 -- src/pnld-download/index.ts | 27 - src/pnld-download/processPnldFiles.ts | 72 --- 6 files changed, 3 insertions(+), 1057 deletions(-) delete mode 100644 src/pnld-download/PnldFileDownloader.ts delete mode 100644 src/pnld-download/config.ts delete mode 100644 src/pnld-download/convertXml.ts delete mode 100644 src/pnld-download/index.ts delete mode 100644 src/pnld-download/processPnldFiles.ts diff --git a/package-lock.json b/package-lock.json index f22cc5eb..9ee724a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,10 +9,9 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@aws-sdk/client-s3": "^3.658.1", - "@types/jest": "^29.5.13", - "@types/lodash": "^4.17.9", - "@types/puppeteer": "^7.0.4", + "@aws-sdk/client-s3": "^3.645.0", + "@types/jest": "^29.5.12", + "@types/lodash": "^4.17.7", "axios": "^1.7.7", "date-fns": "^4.1.0", "eslint": "^8.57.1", @@ -32,7 +31,6 @@ "mockdate": "^3.0.5", "npm-check-updates": "^17.1.3", "prettier": "^3.3.3", - "puppeteer": "^23.4.1", "sort-json-keys": "^1.0.3", "ts-jest": "^29.2.5", "ts-node": "^10.9.2", @@ -2037,27 +2035,6 @@ "url": "https://opencollective.com/unts" } }, - "node_modules/@puppeteer/browsers": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.4.0.tgz", - "integrity": "sha512-x8J1csfIygOwf6D6qUAZ0ASk3z63zPb7wkNeHRerCMh82qWKUrOgkuP005AJC8lDL6/evtXETGEJVcwykKT4/g==", - "dependencies": { - "debug": "^4.3.6", - "extract-zip": "^2.0.1", - "progress": "^2.0.3", - "proxy-agent": "^6.4.0", - "semver": "^7.6.3", - "tar-fs": "^3.0.6", - "unbzip2-stream": "^1.4.3", - "yargs": "^17.7.2" - }, - "bin": { - "browsers": "lib/cjs/main-cli.js" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/@rtsao/scc": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", @@ -2736,11 +2713,6 @@ "node": ">=16.0.0" } }, - "node_modules/@tootallnate/quickjs-emscripten": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", - "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" - }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -2854,15 +2826,6 @@ "undici-types": "~6.19.2" } }, - "node_modules/@types/puppeteer": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-7.0.4.tgz", - "integrity": "sha512-ja78vquZc8y+GM2al07GZqWDKQskQXygCDiu0e3uO0DMRKqE0MjrFBFmTulfPYzLB6WnL7Kl2tFPy0WXSpPomg==", - "deprecated": "This is a stub types definition. puppeteer provides its own type definitions, so you do not need this installed.", - "dependencies": { - "puppeteer": "*" - } - }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -2887,15 +2850,6 @@ "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==" }, - "node_modules/@types/yauzl": { - "version": "2.10.3", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", - "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", - "optional": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.18.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.18.0.tgz", @@ -3338,17 +3292,6 @@ "node": ">=8" } }, - "node_modules/ast-types": { - "version": "0.13.4", - "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", - "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", - "dependencies": { - "tslib": "^2.0.1" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/async": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", @@ -3383,11 +3326,6 @@ "proxy-from-env": "^1.1.0" } }, - "node_modules/b4a": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", - "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==" - }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -3502,48 +3440,6 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, - "node_modules/bare-events": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.0.tgz", - "integrity": "sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==", - "optional": true - }, - "node_modules/bare-fs": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.5.tgz", - "integrity": "sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==", - "optional": true, - "dependencies": { - "bare-events": "^2.0.0", - "bare-path": "^2.0.0", - "bare-stream": "^2.0.0" - } - }, - "node_modules/bare-os": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.4.tgz", - "integrity": "sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==", - "optional": true - }, - "node_modules/bare-path": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", - "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", - "optional": true, - "dependencies": { - "bare-os": "^2.1.0" - } - }, - "node_modules/bare-stream": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.3.0.tgz", - "integrity": "sha512-pVRWciewGUeCyKEuRxwv06M079r+fRjAQjBEK2P6OYGrO43O+Z0LrPZZEjlc4mB6C2RpZ9AxJ1s7NLEtOHO6eA==", - "optional": true, - "dependencies": { - "b4a": "^1.6.6", - "streamx": "^2.20.0" - } - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -3563,14 +3459,6 @@ } ] }, - "node_modules/basic-ftp": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", - "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/bignumber.js": { "version": "9.1.2", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", @@ -3653,37 +3541,6 @@ "node-int64": "^0.4.0" } }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "engines": { - "node": "*" - } - }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -3782,19 +3639,6 @@ "node": ">=10" } }, - "node_modules/chromium-bidi": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.6.5.tgz", - "integrity": "sha512-RuLrmzYrxSb0s9SgpB+QN5jJucPduZQ/9SIe76MDxYJuecPW5mxMdacJ1f4EtgiV+R0p3sCkznTMvH0MPGFqjA==", - "dependencies": { - "mitt": "3.0.1", - "urlpattern-polyfill": "10.0.0", - "zod": "3.23.8" - }, - "peerDependencies": { - "devtools-protocol": "*" - } - }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", @@ -3975,31 +3819,6 @@ "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" }, - "node_modules/cosmiconfig": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", - "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", - "dependencies": { - "env-paths": "^2.2.1", - "import-fresh": "^3.3.0", - "js-yaml": "^4.1.0", - "parse-json": "^5.2.0" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/d-fischer" - }, - "peerDependencies": { - "typescript": ">=4.9.5" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, "node_modules/crc-32": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", @@ -4049,14 +3868,6 @@ "node": ">= 8" } }, - "node_modules/data-uri-to-buffer": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", - "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", - "engines": { - "node": ">= 14" - } - }, "node_modules/data-view-buffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.1.tgz", @@ -4188,19 +3999,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/degenerator": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", - "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", - "dependencies": { - "ast-types": "^0.13.4", - "escodegen": "^2.1.0", - "esprima": "^4.0.1" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -4217,11 +4015,6 @@ "node": ">=8" } }, - "node_modules/devtools-protocol": { - "version": "0.0.1342118", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1342118.tgz", - "integrity": "sha512-75fMas7PkYNDTmDyb6PRJCH7ILmHLp+BhrZGeMsa4bCh40DTxgCz2NRy5UDzII4C5KuD0oBMZ9vXKhEl6UD/3w==" - }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -4303,22 +4096,6 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.3.0.tgz", "integrity": "sha512-QpLs9D9v9kArv4lfDEgg1X/gN5XLnf/A6l9cs8SPZLRZR3ZkY9+kwIQTxm+fsSej5UMYGE8fdoaZVIBlqG0XTw==" }, - "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/env-paths": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", - "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", - "engines": { - "node": ">=6" - } - }, "node_modules/environment": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", @@ -4483,26 +4260,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/escodegen": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", - "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^5.2.0", - "esutils": "^2.0.2" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=6.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, "node_modules/eslint": { "version": "8.57.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", @@ -4949,39 +4706,6 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, - "node_modules/extract-zip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", - "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "dependencies": { - "debug": "^4.1.1", - "get-stream": "^5.1.0", - "yauzl": "^2.10.0" - }, - "bin": { - "extract-zip": "cli.js" - }, - "engines": { - "node": ">= 10.17.0" - }, - "optionalDependencies": { - "@types/yauzl": "^2.9.1" - } - }, - "node_modules/extract-zip/node_modules/get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "dependencies": { - "pump": "^3.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4993,11 +4717,6 @@ "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", "dev": true }, - "node_modules/fast-fifo": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", - "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" - }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", @@ -5076,14 +4795,6 @@ "bser": "2.1.1" } }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "dependencies": { - "pend": "~1.2.0" - } - }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -5206,19 +4917,6 @@ "node": ">=0.8" } }, - "node_modules/fs-extra": { - "version": "11.2.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", - "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -5376,20 +5074,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-uri": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", - "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", - "dependencies": { - "basic-ftp": "^5.0.2", - "data-uri-to-buffer": "^6.0.2", - "debug": "^4.3.4", - "fs-extra": "^11.2.0" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -5844,29 +5528,6 @@ "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==" }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/http-proxy-agent/node_modules/agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dependencies": { - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -5901,25 +5562,6 @@ "url": "https://github.com/sponsors/typicode" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/ignore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", @@ -5997,18 +5639,6 @@ "node": ">= 0.4" } }, - "node_modules/ip-address": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", - "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", - "dependencies": { - "jsbn": "1.1.0", - "sprintf-js": "^1.1.3" - }, - "engines": { - "node": ">= 12" - } - }, "node_modules/is-array-buffer": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz", @@ -6951,11 +6581,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/jsbn": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", - "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" - }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -7006,17 +6631,6 @@ "node": ">=6" } }, - "node_modules/jsonfile": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", - "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, "node_modules/jsonlint-mod": { "version": "1.7.6", "resolved": "https://registry.npmjs.org/jsonlint-mod/-/jsonlint-mod-1.7.6.tgz", @@ -7602,11 +7216,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/mitt": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", - "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==" - }, "node_modules/mockdate": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/mockdate/-/mockdate-3.0.5.tgz", @@ -7622,14 +7231,6 @@ "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==" }, - "node_modules/netmask": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", - "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -7868,59 +7469,6 @@ "node": ">=6" } }, - "node_modules/pac-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", - "integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==", - "dependencies": { - "@tootallnate/quickjs-emscripten": "^0.23.0", - "agent-base": "^7.0.2", - "debug": "^4.3.4", - "get-uri": "^6.0.1", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.5", - "pac-resolver": "^7.0.1", - "socks-proxy-agent": "^8.0.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/pac-proxy-agent/node_modules/agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dependencies": { - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", - "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", - "dependencies": { - "agent-base": "^7.0.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/pac-resolver": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", - "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", - "dependencies": { - "degenerator": "^5.0.0", - "netmask": "^2.0.2" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7986,11 +7534,6 @@ "node": ">=8" } }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" - }, "node_modules/picocolors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", @@ -8151,14 +7694,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -8171,69 +7706,11 @@ "node": ">= 6" } }, - "node_modules/proxy-agent": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", - "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", - "dependencies": { - "agent-base": "^7.0.2", - "debug": "^4.3.4", - "http-proxy-agent": "^7.0.1", - "https-proxy-agent": "^7.0.3", - "lru-cache": "^7.14.1", - "pac-proxy-agent": "^7.0.1", - "proxy-from-env": "^1.1.0", - "socks-proxy-agent": "^8.0.2" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/proxy-agent/node_modules/agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dependencies": { - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/proxy-agent/node_modules/https-proxy-agent": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", - "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", - "dependencies": { - "agent-base": "^7.0.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/proxy-agent/node_modules/lru-cache": { - "version": "7.18.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", - "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", - "engines": { - "node": ">=12" - } - }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, - "node_modules/pump": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", - "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -8242,42 +7719,6 @@ "node": ">=6" } }, - "node_modules/puppeteer": { - "version": "23.4.1", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-23.4.1.tgz", - "integrity": "sha512-+wWfWTkQ8L9IB/3OVGSUp37c0eQ5za/85KdX+LAq2wTZkMdocgYGMCs+/91e2f/RXIYzve4x/uGxN8zG2sj8+w==", - "hasInstallScript": true, - "dependencies": { - "@puppeteer/browsers": "2.4.0", - "chromium-bidi": "0.6.5", - "cosmiconfig": "^9.0.0", - "devtools-protocol": "0.0.1342118", - "puppeteer-core": "23.4.1", - "typed-query-selector": "^2.12.0" - }, - "bin": { - "puppeteer": "lib/cjs/puppeteer/node/cli.js" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/puppeteer-core": { - "version": "23.4.1", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.4.1.tgz", - "integrity": "sha512-uCxGtn8VE9PlKhdFJX/zZySi9K3Ufr3qUZe28jxJoZUqiMJOi+SFh2zhiFDSjWqZIDkc0FtnaCC+rewW3MYXmg==", - "dependencies": { - "@puppeteer/browsers": "2.4.0", - "chromium-bidi": "0.6.5", - "debug": "^4.3.7", - "devtools-protocol": "0.0.1342118", - "typed-query-selector": "^2.12.0", - "ws": "^8.18.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -8326,11 +7767,6 @@ } ] }, - "node_modules/queue-tick": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", - "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" - }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", @@ -8676,52 +8112,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/smart-buffer": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", - "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "engines": { - "node": ">= 6.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", - "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", - "dependencies": { - "ip-address": "^9.0.5", - "smart-buffer": "^4.2.0" - }, - "engines": { - "node": ">= 10.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks-proxy-agent": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", - "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", - "dependencies": { - "agent-base": "^7.1.1", - "debug": "^4.3.4", - "socks": "^2.8.3" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/socks-proxy-agent/node_modules/agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dependencies": { - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/sort-json-keys": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sort-json-keys/-/sort-json-keys-1.0.3.tgz", @@ -8750,11 +8140,6 @@ "source-map": "^0.6.0" } }, - "node_modules/sprintf-js": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", - "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" - }, "node_modules/ssf": { "version": "0.11.2", "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", @@ -8785,19 +8170,6 @@ "node": ">=8" } }, - "node_modules/streamx": { - "version": "2.20.1", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.1.tgz", - "integrity": "sha512-uTa0mU6WUC65iUvzKH4X9hEdvSW7rbPxPtwfWiLMSj3qTdQbAiUboZTxauKfpFuGIGa1C2BYijZ7wgdUXICJhA==", - "dependencies": { - "fast-fifo": "^1.3.2", - "queue-tick": "^1.0.1", - "text-decoder": "^1.1.0" - }, - "optionalDependencies": { - "bare-events": "^2.2.0" - } - }, "node_modules/string-argv": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz", @@ -8986,29 +8358,6 @@ "url": "https://opencollective.com/unts" } }, - "node_modules/tar-fs": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", - "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", - "dependencies": { - "pump": "^3.0.0", - "tar-stream": "^3.1.5" - }, - "optionalDependencies": { - "bare-fs": "^2.1.1", - "bare-path": "^2.1.0" - } - }, - "node_modules/tar-stream": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", - "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", - "dependencies": { - "b4a": "^1.6.4", - "fast-fifo": "^1.2.0", - "streamx": "^2.15.0" - } - }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -9042,24 +8391,11 @@ "node": "*" } }, - "node_modules/text-decoder": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.0.tgz", - "integrity": "sha512-n1yg1mOj9DNpk3NeZOx7T6jchTbyJS3i3cucbNN6FcdPriMZx7NsgrGpWWdWZZGxD7ES1XB+3uoqHMgOKaN+fg==", - "dependencies": { - "b4a": "^1.6.4" - } - }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==" }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" - }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -9323,11 +8659,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/typed-query-selector": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", - "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==" - }, "node_modules/typescript": { "version": "5.6.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", @@ -9354,15 +8685,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/unbzip2-stream": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", - "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", - "dependencies": { - "buffer": "^5.2.1", - "through": "^2.3.8" - } - }, "node_modules/underscore": { "version": "1.13.7", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz", @@ -9373,14 +8695,6 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.6.tgz", "integrity": "sha512-e/vggGopEfTKSvj4ihnOLTsqhrKRN3LeO6qSN/GxohhuRv8qH9bNQ4B8W7e/vFL+0XTnmHPB4/kegunZGA4Org==" }, - "node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", - "engines": { - "node": ">= 10.0.0" - } - }, "node_modules/update-browserslist-db": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", @@ -9423,11 +8737,6 @@ "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==" }, - "node_modules/urlpattern-polyfill": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz", - "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==" - }, "node_modules/uuid": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", @@ -9620,26 +8929,6 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, - "node_modules/ws": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", - "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/xlsx": { "version": "0.18.5", "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", @@ -9735,15 +9024,6 @@ "node": ">=8" } }, - "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", - "dependencies": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } - }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/src/pnld-download/PnldFileDownloader.ts b/src/pnld-download/PnldFileDownloader.ts deleted file mode 100644 index 2bb8e1ac..00000000 --- a/src/pnld-download/PnldFileDownloader.ts +++ /dev/null @@ -1,151 +0,0 @@ -/* eslint-disable no-await-in-loop */ -import * as fs from "fs" -import puppeteer, { ElementHandle, Browser, Page } from "puppeteer" -import { parse } from "date-fns" -import { PnldConfig } from "./config" - -export type PnldFile = { - date: Date - link?: ElementHandle | null - fileName?: string -} - -export default class PnldFileDownloader { - browser: Browser - - page: Page - - tmpDir: string - - options: PnldConfig - - constructor(options: PnldConfig) { - this.options = options - } - - async setupPuppeteer(): Promise { - this.browser = await puppeteer.launch({ - acceptInsecureCerts: true, - headless: this.options.headless, - args: [ - // Required for Docker version of Puppeteer - "--no-sandbox", - "--disable-setuid-sandbox", - // This will write shared memory files into /tmp instead of /dev/shm, - // because Docker’s default for /dev/shm is 64MB - "--disable-dev-shm-usage", - "--window-size=1024x768" - ] - }) - this.page = await this.browser.newPage() - await this.page.setViewport({ - width: 1024, - height: 768 - }) - } - - async closePuppeteer(): Promise { - this.browser.close() - } - - async setupDownloadFolder(): Promise { - this.tmpDir = `${this.options.tmpDir}/pnld` - if (fs.existsSync(this.tmpDir)) { - await fs.promises.rm(this.tmpDir, { recursive: true }) - } - await fs.promises.mkdir(this.tmpDir, { recursive: true }) - if (this.page) { - // eslint-disable-next-line no-underscore-dangle - await (this.page as any)._client().send("Page.setDownloadBehavior", { - behavior: "allow", - downloadPath: this.tmpDir - }) - } - } - - async login(): Promise { - await this.page.goto(this.options.loginUrl) - await this.page.waitForSelector("button#next") - await this.page.type("#logonIdentifier", this.options.username) - await this.page.type("#password", this.options.password) - return Promise.all([this.page.click("button#next"), this.page.waitForNavigation()]).then( - () => {} - ) - } - - async waitForZipCount(zipCount: number, timeout: number): Promise { - for (let i = 0; i < timeout; i++) { - await new Promise((res) => { - setTimeout(res, 1000) - }) - const dir = await fs.promises.readdir(this.tmpDir) - const count = dir.reduce((acc, filename) => { - if (filename.endsWith(".zip")) { - return acc + 1 - } - return acc - }, 0) - if (count === zipCount) { - return - } - } - } - - async downloadFile(link: ElementHandle): Promise { - const linkText = await link.evaluate((el) => el.textContent) - const linkLocation = await link.evaluate((el) => el.getAttribute("href")) - console.log(`Downloading PNLD file "${linkText}" from ${linkLocation}`) - - const before = await fs.promises.readdir(this.tmpDir) - await link.scrollIntoView() - await link.click() - await this.waitForZipCount(before.length + 1, 60) - const after = await fs.promises.readdir(this.tmpDir) - const fileName = after.filter((f) => !before.includes(f))[0] - const filePath = `${this.tmpDir}/${fileName}` - - console.log(`Downloaded PNLD file "${linkText}" to ${filePath}`) - return filePath - } - - async getFileLinks(): Promise { - const links: PnldFile[] = [] - await this.page.goto(this.options.downloadUrl, { waitUntil: "networkidle2" }) - // The links are in a 4 column table with the date in the 1st column and the link in the second - const columnCount = 4 - const tds = await this.page.$$(".table-responsive table tbody td") - const rowCount = tds.length / columnCount - for (let i = 0; i < rowCount * columnCount; i += columnCount) { - const date = await tds[i].evaluate((node) => (node as any).innerText) - const link = await tds[i + 1].$("a") - links.push({ date: parse(date, "dd/MM/yy", new Date()), link }) - } - return links - } - - async downloadArchives(): Promise { - await this.setupDownloadFolder() - const fileLinks = await this.getFileLinks() - - const cookiesButton = await this.page.$(".closeCookies") - if (cookiesButton) { - await cookiesButton.click() - } - - // eslint-disable-next-line no-restricted-syntax - for (const fileLink of fileLinks) { - if (fileLink.link) { - fileLink.fileName = await this.downloadFile(fileLink.link) - } - } - return fileLinks - } - - async download(): Promise { - await this.setupPuppeteer() - await this.login() - const fileList = await this.downloadArchives() - await this.closePuppeteer() - return fileList.map((f) => ({ date: f.date, fileName: f.fileName })) - } -} diff --git a/src/pnld-download/config.ts b/src/pnld-download/config.ts deleted file mode 100644 index 135c87fb..00000000 --- a/src/pnld-download/config.ts +++ /dev/null @@ -1,35 +0,0 @@ -export type PnldConfig = { - username: string - password: string - loginUrl: string - downloadUrl: string - headless: boolean - tmpDir: string -} - -if (!process.env.PNLD_USERNAME) { - throw new Error("PNLD_USERNAME environment variable is not set") -} - -if (!process.env.PNLD_PASSWORD) { - throw new Error("PNLD_PASSWORD environment variable is not set") -} - -if (!process.env.PNLD_LOGIN_URL) { - throw new Error("PNLD_LOGIN_URL environment variable is not set") -} - -if (!process.env.PNLD_DOWNLOAD_URL) { - throw new Error("PNLD_DOWNLOAD_URL environment variable is not set") -} - -const config: PnldConfig = { - username: process.env.PNLD_USERNAME, - password: process.env.PNLD_PASSWORD, - loginUrl: process.env.PNLD_LOGIN_URL, - downloadUrl: process.env.PNLD_DOWNLOAD_URL, - tmpDir: process.env.TMP_DIR || "./tmp", - headless: process.env.HEADLESS === "true" -} - -export default config diff --git a/src/pnld-download/convertXml.ts b/src/pnld-download/convertXml.ts deleted file mode 100644 index 79add1b5..00000000 --- a/src/pnld-download/convertXml.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { XMLParser } from "fast-xml-parser" -import valueToBoolean from "../lib/valueToBoolean" -import { OffenceCode } from "../types/OffenceCode" - -export type PnldOffenceParsedXml = { - documents: { - document: { - codes: { - cjsoffencecode: string - recordableonpncindicator: { - code: string - } - notifiabletoho: { - code: string - } - hoclassification: string - } - libra: { - cjsoffencecategory: { - code: string - } - } - english: { - title: string - } - } - } -} - -export default async (xmlData: string): Promise => { - const options = { - ignoreAttributes: false, - removeNSPrefix: true - } - - const parser = new XMLParser(options) - const parsedObj = parser.parse(xmlData) as PnldOffenceParsedXml - const doc = parsedObj.documents.document - - return { - cjsCode: doc.codes?.cjsoffencecode, - description: doc.codes?.cjsoffencecode, - offenceTitle: doc.english?.title, - recordableOnPnc: valueToBoolean(doc.codes?.recordableonpncindicator?.code), - notifiableToHo: valueToBoolean(doc.codes?.notifiabletoho?.code), - homeOfficeClassification: doc.codes?.hoclassification, - offenceCategory: doc.libra.cjsoffencecategory?.code - } -} diff --git a/src/pnld-download/index.ts b/src/pnld-download/index.ts deleted file mode 100644 index 9961519d..00000000 --- a/src/pnld-download/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* eslint-disable no-await-in-loop */ -import * as fs from "fs" -import consistentSort from "../lib/consistentSort" -import config from "./config" -import PnldFileDownloader from "./PnldFileDownloader" -import processPnldFiles from "./processPnldFiles" - -export default async () => { - for (let i = 0; i < 5; i++) { - try { - console.log("Downloading PNLD data") - const downloader = new PnldFileDownloader(config) - const files = await downloader.download() - const records = await processPnldFiles(files) - const data = consistentSort(records) - await fs.promises.writeFile( - "input-data/offence-code/pnld-offences.json", - JSON.stringify(data, null, 2) - ) - console.log("PNLD data successfully downloaded") - return - } catch (e) { - console.error("Error downloading PNLD data") - console.error(e) - } - } -} diff --git a/src/pnld-download/processPnldFiles.ts b/src/pnld-download/processPnldFiles.ts deleted file mode 100644 index f476cc94..00000000 --- a/src/pnld-download/processPnldFiles.ts +++ /dev/null @@ -1,72 +0,0 @@ -/* eslint-disable no-await-in-loop, no-restricted-syntax */ -import { exec } from "child_process" -import * as fs from "fs" -import * as util from "util" -import { OffenceCode } from "../types/OffenceCode" -import { PromiseResult } from "../types/Result" -import convertXml from "./convertXml" -import { PnldFile } from "./PnldFileDownloader" - -const execPromise = util.promisify(exec) - -type OffenceCodeMap = { - [index: string]: OffenceCode -} - -const unzipFile = async (file: PnldFile, outDir: string): PromiseResult => { - if (!file.fileName) { - throw new Error("Filename is missing") - } - - if (fs.existsSync(outDir)) { - await fs.promises.rm(outDir, { recursive: true }) - } - await fs.promises.mkdir(outDir) - await execPromise(`unzip -q ${file.fileName} -d ${outDir}`) -} - -const getAllFiles = async (startDir: string): Promise => { - const output = [] - const contents = await fs.promises.readdir(startDir) - for (const file of contents) { - const fullFile = `${startDir}/${file}` - if (file.endsWith(".xml")) { - output.push(fullFile) - } - if (fs.statSync(fullFile).isDirectory()) { - const subDirFiles = await getAllFiles(fullFile) - for (const subFile of subDirFiles) { - output.push(subFile) - } - } - } - return output -} - -const processZip = async (file: PnldFile, output: OffenceCodeMap): Promise => { - if (!file.fileName) { - throw new Error("File name is missing") - } - const outDir = file.fileName.replace(".zip", "") - await unzipFile(file, outDir) - const xmlFiles = await getAllFiles(outDir) - for (const xmlFile of xmlFiles) { - const xmlData = await fs.promises.readFile(xmlFile) - try { - const record = await convertXml(xmlData.toString()) - // eslint-disable-next-line no-param-reassign - output[record.cjsCode] = record - } catch (e) { - console.error("Error processing: ", xmlFile) - throw e - } - } -} - -export default async (files: PnldFile[]): Promise => { - const output: OffenceCodeMap = {} - for (const file of files.reverse()) { - await processZip(file, output) - } - return Object.values(output) -}