From 0673016645d1d538a8c0f23f621fc14c263f4dd2 Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Fri, 9 Aug 2024 19:57:42 -0700 Subject: [PATCH 1/4] combine CI workflows --- .../workflows/{verify_metadata.yml => ci.yml} | 30 ++++++++++++++----- .github/workflows/json_verify.yml | 30 ------------------- 2 files changed, 22 insertions(+), 38 deletions(-) rename .github/workflows/{verify_metadata.yml => ci.yml} (53%) delete mode 100644 .github/workflows/json_verify.yml diff --git a/.github/workflows/verify_metadata.yml b/.github/workflows/ci.yml similarity index 53% rename from .github/workflows/verify_metadata.yml rename to .github/workflows/ci.yml index 403d8e9..65a1ac3 100644 --- a/.github/workflows/verify_metadata.yml +++ b/.github/workflows/ci.yml @@ -1,24 +1,38 @@ -name: Verify Metadata +name: CI on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + branches: [ master ] pull_request: - branches: - - master + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: jobs: - build: + verify_json: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Install yajl-tools + run: sudo apt-get install -qq yajl-tools + + - name: Check that JSON files are valid + run: for f in *.json dist/*.json filter_lists/*.json ; do json_verify < $f || echo "$f failed" ; done + + confirm_built: runs-on: ubuntu-latest steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - with: - fetch-depth: 0 - name: Install node uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 with: - node-version: "18" + node-version: "20" - - name: Get changed files in the docs folder + - name: Get changed resource-related files id: changed-files uses: tj-actions/changed-files@cbda684547adc8c052d50711417fa61b428a9f88 # v41 with: diff --git a/.github/workflows/json_verify.yml b/.github/workflows/json_verify.yml deleted file mode 100644 index 3dd20ea..0000000 --- a/.github/workflows/json_verify.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: CI - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: [ master ] - pull_request: - branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Install yajl-tools - run: sudo apt-get install -qq yajl-tools - - - name: Check that JSON files are valid - run: for f in *.json dist/*.json filter_lists/*.json ; do json_verify < $f || echo "$f failed" ; done From 4ca5028bd73498d69e9372285989e572cd0392cf Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Fri, 9 Aug 2024 19:58:36 -0700 Subject: [PATCH 2/4] add verify.js to CI --- .github/workflows/ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65a1ac3..3ea7d2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,3 +50,25 @@ jobs: if: steps.changed-files.outputs.any_changed == 'true' run: | git diff --no-ext-diff --exit-code metadata.json + + check_format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Install node + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 + with: + node-version: "20" + + - name: See if list catalog has changed + id: changed-files + uses: tj-actions/changed-files@cbda684547adc8c052d50711417fa61b428a9f88 # v41 + with: + files: | + filter_lists/list_catalog.json + + - name: Run format tests + if: steps.changed-files.outputs.any_changed == 'true' + run: | + npm run test From 60ca496bc90d768b70bc474e786d19b850d86c68 Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Fri, 9 Aug 2024 19:59:21 -0700 Subject: [PATCH 3/4] confirm that each component ID corresponds to base64 public key --- verify.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/verify.js b/verify.js index f35d861..d897f3c 100644 --- a/verify.js +++ b/verify.js @@ -1,9 +1,20 @@ const { readResources, listCatalog } = require('.') const assert = require('node:assert') +const crypto = require('crypto') const test = require('node:test') const { Engine, FilterFormat, FilterSet } = require('adblock-rs') +const getIDFromBase64PublicKey = (key) => { + const hash = crypto.createHash('sha256') + const data = Buffer.from(key, 'base64') + const digest = hash.update(data).digest('hex') + const id = digest.toString().substring(0, 32) + return id.replace(/[0-9a-f]/g, (c) => { + return 'abcdefghijklmnop'.charAt('0123456789abcdef'.indexOf(c)) + }) +} + test('resources are parsed OK by adblock-rust', t => { const resources = readResources() @@ -34,6 +45,9 @@ const testLists = (lists) => { assert.ok(list.list_text_component.component_id !== undefined && typeof list.list_text_component.component_id === 'string') assert.ok(list.list_text_component.base64_public_key !== undefined && typeof list.list_text_component.base64_public_key === 'string') + assert.ok(list.component_id === getIDFromBase64PublicKey(list.base64_public_key)) + assert.ok(list.list_text_component.component_id === getIDFromBase64PublicKey(list.list_text_component.base64_public_key)) + assert.ok(list.sources !== undefined && Array.isArray(list.sources)) for (const source of list.sources) { assert.ok(source.url !== undefined && typeof source.url === 'string') From 482b072d6e4df224c41124e8928d3cb2b3aa6228 Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Fri, 9 Aug 2024 20:08:56 -0700 Subject: [PATCH 4/4] migrate to module scripts --- build.js | 8 ++++---- generateMetadataJsonFromScriptResources.js | 4 ++-- index.js | 12 ++++++------ package.json | 1 + verify.js | 10 +++++----- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/build.js b/build.js index 83817c6..964ad66 100644 --- a/build.js +++ b/build.js @@ -1,6 +1,6 @@ -const fs = require('fs') -const path = require('path') +import fs from 'fs' +import path from 'path' -const { readResources } = require('.') +import { readResources } from './index.js' -fs.writeFileSync(path.join(__dirname, 'dist', 'resources.json'), JSON.stringify(readResources())) +fs.writeFileSync(path.join(import.meta.dirname, 'dist', 'resources.json'), JSON.stringify(readResources())) diff --git a/generateMetadataJsonFromScriptResources.js b/generateMetadataJsonFromScriptResources.js index 7dc9e4f..a99aa76 100644 --- a/generateMetadataJsonFromScriptResources.js +++ b/generateMetadataJsonFromScriptResources.js @@ -4,8 +4,8 @@ * For instance - brave-fix script needs the "bf" alias */ -const fs = require("fs"); -const path = require("path"); +import fs from "fs"; +import path from "path"; const metadataJsonFile = "metadata.json"; const resourcesDir = "resources"; diff --git a/index.js b/index.js index 417066a..37bd2c2 100644 --- a/index.js +++ b/index.js @@ -1,17 +1,17 @@ -const fs = require('fs') -const path = require('path') +import fs from 'fs' +import path from 'path' -const metadata = require('./metadata.json') +import metadata from './metadata.json' with { type: "json" } const readResources = (() => { return metadata.map(item => ({ name: item.name, aliases: item.aliases, kind: item.kind, - content: fs.readFileSync(path.join(__dirname, 'resources', item.resourcePath)).toString('base64') + content: fs.readFileSync(path.join(import.meta.dirname, 'resources', item.resourcePath)).toString('base64') })) }) -const listCatalog = require('./filter_lists/list_catalog.json') +import listCatalog from './filter_lists/list_catalog.json' with { type: "json" } -module.exports = { listCatalog, readResources } +export { listCatalog, readResources } diff --git a/package.json b/package.json index 6da3428..4b437be 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "description": "Custom resources and scriptlets used for Brave's adblocker", "main": "index.js", + "type": "module", "scripts": { "build": "node build", "test": "node verify", diff --git a/verify.js b/verify.js index d897f3c..68d75ac 100644 --- a/verify.js +++ b/verify.js @@ -1,9 +1,9 @@ -const { readResources, listCatalog } = require('.') +import { readResources, listCatalog } from './index.js' -const assert = require('node:assert') -const crypto = require('crypto') -const test = require('node:test') -const { Engine, FilterFormat, FilterSet } = require('adblock-rs') +import assert from 'node:assert' +import crypto from 'crypto' +import test from 'node:test' +import { Engine, FilterFormat, FilterSet } from 'adblock-rs' const getIDFromBase64PublicKey = (key) => { const hash = crypto.createHash('sha256')