From eed578e28a70d62f430667523cb008fa3c345f81 Mon Sep 17 00:00:00 2001 From: Sameer Srivastava Date: Sun, 7 Apr 2024 21:10:40 +0200 Subject: [PATCH] Issue_1993 + Issue_2057 --- .github/workflows/benchmarks.yml | 143 +++++++++++ .github/workflows/pullrequest_check.yml | 227 ++++++++++++------ .github/workflows/release.yml | 2 +- .../rustcore/ts-bindings/spec/benchmarks.json | 138 +++++++++++ .../apps/rustcore/ts-bindings/spec/common.ts | 44 ++++ .../apps/rustcore/ts-bindings/spec/config.ts | 12 + .../rustcore/ts-bindings/spec/defaults.json | 50 +++- .../spec/session.benchmark.spec.ts | 9 + .../ts-bindings/spec/session.indexes.spec.ts | 119 ++++++++- .../ts-bindings/spec/session.observe.spec.ts | 36 ++- .../ts-bindings/spec/session.search.spec.ts | 146 ++++++++++- .../ts-bindings/spec/session.stream.spec.ts | 126 +++++++++- .../rustcore/ts-bindings/spec/setup_config.sh | 9 +- scripts/elements/bindings.rb | 14 +- scripts/tools/run_benchmarks.rb | 213 ++++++++++++++++ scripts/tools/run_benchmarks1.rb | 208 ++++++++++++++++ 16 files changed, 1402 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 application/apps/rustcore/ts-bindings/spec/benchmarks.json create mode 100644 application/apps/rustcore/ts-bindings/spec/session.benchmark.spec.ts mode change 100644 => 100755 application/apps/rustcore/ts-bindings/spec/setup_config.sh create mode 100644 scripts/tools/run_benchmarks.rb create mode 100644 scripts/tools/run_benchmarks1.rb diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..4312dce969 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,143 @@ +name: Benchmarks + +on: [workflow_dispatch] + +jobs: + # invoke_jasmine_tests: + # name: Invoke Jasmine performance tests + # runs-on: sh-linux-x86-chipmunk # Specify that it runs on a self-hosted runner + # steps: + # - name: Checkout + # uses: actions/checkout@v3 + # - name: libudev-dev and install Ruby + # run: | + # whoami + # sudo yum install -y libudev-devel ruby-devel cargo npm + # sudo yum install rbenv + # rbenv install 3.1.2 -s + # rbenv global 3.1.2 + # sudo chown -R $(whoami) /usr/local + # - name: install ruby:gem::dotenv + # run: sudo gem install dotenv + # - name: install ruby:gem::json + # run: sudo gem install json + # - name: install node + # uses: actions/setup-node@master + # with: + # node-version: "current" + # - name: install rust + # uses: hecrj/setup-rust-action@v1 + # with: + # rust-version: stable + # - name: cargo install wasm-pack + # run: | + # pwd + # whoami + # cargo install wasm-pack --locked + # npm install -y yarn rustup + # export PATH="$HOME/.cargo/bin:$PATH" + # export PATH="$HOME/.cargo/env:$PATH" + # . "$HOME/.cargo/env" + # - name: Run Jasmine performance tests + # run: | + # source ./application/apps/rustcore/ts-bindings/spec/setup_config.sh sample_tag + # pwd + # cargo install nj-cli --locked + # source ~/.bashrc + # rustup -V + # whoami + # nj-cli --version + # printenv + # rake bindings:clean + # rake bindings:test:indexes + # rake bindings:test:search + # rake bindings:test:observe + invoke_jasmine_tests: + name: Invoke Jasmine performance tests + runs-on: self-hosted-runner-ubuntu + steps: + - name: 'Cleanup build folder' + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ + - name: Checkout + uses: actions/checkout@v3 + - name: libudev-dev + run: | + sudo apt-get update && sudo apt-get install -y gconf-service libgbm-dev libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libnss3 lsb-release xdg-utils wget ca-certificates + sudo apt-get install -y libudev-dev cargo npm imagemagick libmagickwand-dev cmake + - name: install node + uses: actions/setup-node@v4 + with: + node-version: 22.4.0 + - name: Install Ruby and required gems + run: | + whoami + npm install -g corepack + corepack enable + curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - + echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list + apt-get update + sudo apt-get install -y yarn + yarn --version + apt update + apt-get install -y software-properties-common + apt-add-repository -y ppa:rael-gc/rvm + apt-get update + apt-get install -y rvm + echo 'source "/etc/profile.d/rvm.sh"' >> ~/.bashrc + source /etc/profile.d/rvm.sh + which rvm + rvm install ruby-3.1.2 + ruby --version + sudo chown -R $(whoami) /usr/local + rvm use 3.1.2 --default + rvm --version + rvm info + which rvm + ruby --version + - name: cargo install nj-cli + run: | + cargo install nj-cli --locked + cargo install wasm-pack --locked + - name: Run Jasmine performance tests + run: | + npm i -g tslib + export PATH="/usr/share/rvm:$PATH" + which ruby + ruby --version + pwd + gem install dotenv json octokit tmpdir fileutils + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + export PATH="/root/.cargo/bin:$PATH" + . "/root/.cargo/env" + source ~/.bashrc + ruby scripts/tools/run_benchmarks.rb 10 + ls -la /chipmunk/chipmunk_performance_results + env: + REPO_OWNER: 'esrlabs' + REPO_NAME: 'chipmunk' + invoke_package_distribution: + name: Move benchmark data to chipmunk-docs repository + needs: invoke_jasmine_tests + runs-on: self-hosted-runner-ubuntu + steps: + - name: Checkout chipmunk-docs repository + uses: actions/checkout@v2 + with: + repository: esrlabs/chipmunk-docs + path: './chipmunk-docs' + token: ${{secrets.DOCS_PUSH_TOKEN}} + - name: Push tag + working-directory: ./chipmunk-docs + run: | + ls -la + cp /chipmunk/chipmunk_performance_results/data.json ./jekyll/benchmarks/data/data.json + git config user.name "esrlabs" + git config user.email "esrlabs@gmail.com" + git remote set-url origin "https://esrlabs:${{secrets.DOCS_PUSH_TOKEN}}@github.com/esrlabs/chipmunk-docs" + git add ./jekyll/benchmarks/data/data.json + git commit -m "Updating data.json for latest tag" + git push origin master diff --git a/.github/workflows/pullrequest_check.yml b/.github/workflows/pullrequest_check.yml index 833bb91546..d3e3373f60 100644 --- a/.github/workflows/pullrequest_check.yml +++ b/.github/workflows/pullrequest_check.yml @@ -3,87 +3,158 @@ name: Checks on: [pull_request] jobs: - ts_lint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: install ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.0" - bundler-cache: true - - name: install ruby:gem::dotenv - run: gem install dotenv - - name: install ruby:gem::json - run: gem install json - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - name: enable corepack for yarnpkg upgrade - run: corepack enable - - name: JS/TS linting - run: rake lint:js - rust_lint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: libudev-dev - run: sudo apt-get install -y libudev-dev - - name: install ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.0" - bundler-cache: true - - name: install ruby:gem::dotenv - run: gem install dotenv - - name: install ruby:gem::json - run: gem install json - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - name: Rust linting - run: rake lint:rust - integration_tests: - runs-on: ubuntu-latest + # ts_lint: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v2 + # - name: install ruby + # uses: ruby/setup-ruby@v1 + # with: + # ruby-version: "3.0" + # bundler-cache: true + # - name: install ruby:gem::dotenv + # run: gem install dotenv + # - name: install ruby:gem::json + # run: gem install json + # - uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true + # - name: enable corepack for yarnpkg upgrade + # run: corepack enable + # - name: JS/TS linting + # run: rake lint:js + # rust_lint: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v2 + # - name: libudev-dev + # run: sudo apt-get install -y libudev-dev + # - name: install ruby + # uses: ruby/setup-ruby@v1 + # with: + # ruby-version: "3.0" + # bundler-cache: true + # - name: install ruby:gem::dotenv + # run: gem install dotenv + # - name: install ruby:gem::json + # run: gem install json + # - uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true + # - name: Rust linting + # run: rake lint:rust + # integration_tests: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v2 + # - name: libudev-dev + # run: sudo apt-get install -y libudev-dev + # - name: install ruby + # uses: ruby/setup-ruby@v1 + # with: + # ruby-version: "3.0" + # bundler-cache: true + # - name: install ruby:gem::dotenv + # run: gem install dotenv + # - name: install ruby:gem::json + # run: gem install json + # - uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true + # - name: enable corepack for yarnpkg upgrade + # run: | + # npm install tslib + # corepack enable + # - name: Run integration tests + # run: rake test:js + # unit_tests: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v2 + # - name: libudev-dev + # run: sudo apt-get install -y libudev-dev + # - name: install ruby + # uses: ruby/setup-ruby@v1 + # with: + # ruby-version: "3.0" + # bundler-cache: true + # - name: Run unit tests on indexer + # run: rake test:rust + benchmarks: + name: Invoke Jasmine performance tests + runs-on: self-hosted-runner-ubuntu steps: + - name: 'Cleanup build folder' + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: libudev-dev - run: sudo apt-get install -y libudev-dev - - name: install ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.0" - bundler-cache: true - - name: install ruby:gem::dotenv - run: gem install dotenv - - name: install ruby:gem::json - run: gem install json - - uses: actions-rs/toolchain@v1 + run: | + sudo apt-get update && sudo apt-get install -y gconf-service libgbm-dev libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libnss3 lsb-release xdg-utils wget ca-certificates + sudo apt-get install -y libudev-dev cargo npm imagemagick libmagickwand-dev cmake + - name: install node + uses: actions/setup-node@v4 with: - toolchain: stable - override: true - - name: enable corepack for yarnpkg upgrade + node-version: 22.4.0 + - name: Install Ruby and required gems run: | - npm install tslib + whoami + npm install -g corepack corepack enable - - name: Run integration tests - run: rake test:js - unit_tests: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: libudev-dev - run: sudo apt-get install -y libudev-dev - - name: install ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.0" - bundler-cache: true - - name: Run unit tests on indexer - run: rake test:rust + curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - + echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list + apt-get update + sudo apt-get install -y yarn + yarn --version + apt update + apt-get install -y software-properties-common + apt-add-repository -y ppa:rael-gc/rvm + apt-get update + apt-get install -y rvm + echo 'source "/etc/profile.d/rvm.sh"' >> ~/.bashrc + source /etc/profile.d/rvm.sh + which rvm + rvm install ruby-3.1.2 + ruby --version + sudo chown -R $(whoami) /usr/local + rvm use 3.1.2 --default + rvm --version + rvm info + which rvm + ruby --version + - name: cargo install nj-cli + run: | + cargo install nj-cli --locked + cargo install wasm-pack --locked + - name: Run Jasmine performance tests + run: | + npm i -g tslib + export PATH="/usr/share/rvm:$PATH" + which ruby + ruby --version + pwd + gem install dotenv json octokit tmpdir fileutils + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + export PATH="/root/.cargo/bin:$PATH" + . "/root/.cargo/env" + source ~/.bashrc + echo "PR head repo: ${{ github.event.pull_request.head.repo.name }}" + echo "PR head branch: ${{ github.event.pull_request.head.ref }}" + echo "PR head owner: ${{ github.event.pull_request.head.repo.owner.login }}" + ls -la /chipmunk/chipmunk_performance_results + printenv + ruby scripts/tools/run_benchmarks1.rb PR~${{ github.event.pull_request.number }} + env: + REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + REPO_NAME: ${{ github.event.pull_request.head.repo.name }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d3a557e7b8..faafb2a367 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -134,4 +134,4 @@ jobs: git config user.email "esrlabs@gmail.com" git remote set-url origin "https://esrlabs:${{secrets.PUSH_TOKEN}}@github.com/esrlabs/chipmunk-distribution" git tag ${{ github.ref_name }} - git push origin ${{ github.ref_name }} \ No newline at end of file + git push origin ${{ github.ref_name }} diff --git a/application/apps/rustcore/ts-bindings/spec/benchmarks.json b/application/apps/rustcore/ts-bindings/spec/benchmarks.json new file mode 100644 index 0000000000..b0f5bf4d8d --- /dev/null +++ b/application/apps/rustcore/ts-bindings/spec/benchmarks.json @@ -0,0 +1,138 @@ +/* NOTE: in this JSON can be used comments placed between stars */ +{ + "log_level": 1, + "tests": { + "observe": { + "regular": { + /* With numbers you can define, which tests should be executed. "execute_only": [1, 3] will run ONLY tests 1 and 3*/ + /* If "execute_only" isn't empty, all performance tests will be ignored*/ + "execute_only": [-1], + "list": { + "1": "Test 1. Observe and grab content (text)", + "2": "Test 2. Observe and grab content (pcapng)", + "3": "Test 3. Observe and grab content (dlt)", + "4": "Test 4. Observe and grab content (attachments)", + "5": "Test 5. Observe and grab content (someip from pcapng)", + "6": "Test 6. Observe and grab content (someip from pcapng with fibex)", + "7": "Test 7. Observe and grab content (someip from pcap)", + "8": "Test 8. Observe and grab content (someip from pcap with fibex)" + }, + "files": { + } + }, + "performance": { + /* In false will prevent running performance tests */ + "run": true, + "tests": { + "test1": { + "alias": "Observe - grab content (text)", + "open_as": "text", + "file": "test_files/temp_readings3.txt", + "expectation_ms": 10000 + }, + "test2": { + "alias": "Observe - grab content (pcapng)", + "open_as": "dlt", + "file": "test_files/FzgProg_SP21.dlt", + "expectation_ms": 60000 + }, + "test3": { + "alias": "Observe - grab content (dlt)", + "open_as": "pcapng", + "file": "test_files/someip.pcapng", + "expectation_ms": 1000 + } + } + } + }, + "stream": { + "regular": { + "execute_only": [-1], + "list": { + "1": "Test 1. Observe and grab", + "2": "Test 2. Life cycle", + "3": "Test 3. Invalid data source", + "4": "Test 4. Updated stream", + "5": "Test 5. Updated stream search", + "6": "Test 6. Aborting stream", + "7": "Test 7. Multiple stream & SDE test" + }, + "files": { + } + }, + "performance": { + "run": true, + "tests": { + "test1": { + "alias": "Stream - startup measurement", + "file": "", + "expectation_ms": 1000 + }, + "test2": { + "alias": "Stream - shutdown measurement", + "file": "", + "expectation_ms": 10000 + }, + "test3": { + "alias": "Stream - Open 50 sessions", + "file": "", + "expectation_ms": 10000 + } + } + } + }, + "indexes": { + "regular": { + "execute_only": [-1], + "list": { + "1": "Test 1. Switch to breadcrumb mode" + }, + "files": { + } + }, + "performance": { + /* In false will prevent running performance tests */ + "run": true, + "tests": { + "test1": { + "alias": "Indexes - Switch to breadcrumb mode", + "file": "test_files/indexing_access_huge.log", + "expectation_ms": 15000 + } + } + } + }, + "search": { + "regular": { + "execute_only": [-1], + "list": { + "1": "Test 1. Assign & single search", + "2": "Test 2. Assign & multiple search", + "3": "Test 3. Assign & zero search", + "4": "Test 4. Assign & single not case sensitive search", + "5": "Test 5. Assign & single word search", + "6": "Test 6. Assign & single search with crossing terms", + "7": "Test 7. Assign & repeated search" + }, + "files": { + } + }, + "performance": { + /* In false will prevent running performance tests */ + "run": true, + "tests": { + "test1": { + "alias": "Assign & single search", + "file": "test_files/indexing_access_huge.log", + "expectation_ms": 1000 + }, + "test2": { + "alias": "Assign & multiple search", + "file": "test_files/indexing_access_huge.log", + "expectation_ms": 10000 + } + } + } + } + } +} diff --git a/application/apps/rustcore/ts-bindings/spec/common.ts b/application/apps/rustcore/ts-bindings/spec/common.ts index 342a34aa49..50a930abf5 100644 --- a/application/apps/rustcore/ts-bindings/spec/common.ts +++ b/application/apps/rustcore/ts-bindings/spec/common.ts @@ -12,6 +12,8 @@ import { IRegularTests } from './config'; import * as tmp from 'tmp'; import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; const NS_PER_SEC = 1e9; const NS_PER_MS = 1000000; @@ -208,6 +210,44 @@ export function performanceReport( ), ); output(`└${'─'.repeat(LEN)}┘`); + + const result = { + name, + actual, + expectation, + passed: actual <= expectation, + }; + + let performance_results_folder = (process.env as any)['PERFORMANCE_RESULTS_FOLDER']; + let performance_results = (process.env as any)['PERFORMANCE_RESULTS']; + let home_dir = (process.env as any)['SH_HOME_DIR']; + if (home_dir && performance_results_folder) { + const folderPath = path.join(home_dir, performance_results_folder); + const filePath = path.join(folderPath, performance_results); + // Ensure filePath is a real path + if (!fs.existsSync(folderPath)) { + // Create directory if it doesn't exist + fs.mkdirSync(folderPath, { recursive: true }); + output(`Created directory: ${folderPath}`); + } + + let results = []; + if (fs.existsSync(filePath)) { + let existingData = fs.readFileSync(filePath, 'utf-8'); + try { + results = JSON.parse(existingData); + } catch (error) { + output('Error parsing existing JSON data:'); + } + } + results.push(result); + const data = JSON.stringify(results, null, 2); // JSON format with indentation + fs.writeFileSync(filePath, data); + } else { + output(`Missing necessary environment variables for file path.`); + } + + return actual <= expectation; } @@ -231,3 +271,7 @@ export function setMeasurement(): () => ITimeMeasurement { }; }; } + +export function helloWorld() { + return 'Hello, World!'; +} diff --git a/application/apps/rustcore/ts-bindings/spec/config.ts b/application/apps/rustcore/ts-bindings/spec/config.ts index 6bed31a356..ebd1ed0ffa 100644 --- a/application/apps/rustcore/ts-bindings/spec/config.ts +++ b/application/apps/rustcore/ts-bindings/spec/config.ts @@ -43,9 +43,17 @@ export interface IConfiguration { }; stream: { regular: IRegularTests; + performance: { + run: boolean; + tests: { [key: string]: IPerformanceTest }; + }; }; indexes: { regular: IRegularTests; + performance: { + run: boolean; + tests: { [key: string]: IPerformanceTest }; + }; }; jobs: { regular: IRegularTests; @@ -55,6 +63,10 @@ export interface IConfiguration { }; search: { regular: IRegularTests; + performance: { + run: boolean; + tests: { [key: string]: IPerformanceTest }; + }; }; values: { regular: IRegularTests; diff --git a/application/apps/rustcore/ts-bindings/spec/defaults.json b/application/apps/rustcore/ts-bindings/spec/defaults.json index d616406765..fc061a4744 100644 --- a/application/apps/rustcore/ts-bindings/spec/defaults.json +++ b/application/apps/rustcore/ts-bindings/spec/defaults.json @@ -18,8 +18,8 @@ "8": "Test 8. Observe and grab content (someip from pcap with fibex)" }, "files": { - "pcapng": "/storage/projects/esrlabs/logs-examples/test.pcapng", - "dlt": "/storage/projects/esrlabs/logs-examples/dlt_receive_2.dlt", + "pcapng": "../../../../application/developing/resources/many_interfaces.pcapng", + "dlt": "../../../../application/developing/resources/DTC_SP21.dlt", "attachments": "../../../../application/developing/resources/attachments.dlt", "someip-pcapng": "../../../../application/developing/resources/someip.pcapng", "someip-pcap": "../../../../application/developing/resources/someip.pcap", @@ -28,8 +28,26 @@ }, "performance": { /* In false will prevent running performance tests */ - "run": false, + "run": true, "tests": { + "test1": { + "alias": "Performance Test 1", + "open_as": "text", + "file": "../../../../application/developing/resources/temp_readings3.txt", + "expectation_ms": 1000 + }, + "test2": { + "alias": "Performance Test 2", + "open_as": "dlt", + "file": "../../../../application/developing/resources/FzgProg_SP21.dlt", + "expectation_ms": 10000 + }, + "test3": { + "alias": "Performance Test you 3", + "open_as": "pcapng", + "file": "../../../../application/developing/resources/many_interfaces.pcapng", + "expectation_ms": 500 + } } } }, @@ -70,7 +88,8 @@ "5": "Test 5. Get envvars", "6": "Test 6. Get Someip statistic", "7": "Test 7. Check if file is binary", - "8": "Test 8. Closing session with running task" + "8": "Test 8. Closing session with running task", + "9": "Test 6. Get DLT statistic" }, "files": { "someip-pcapng": "../../../../application/developing/resources/someip.pcapng", @@ -111,6 +130,29 @@ "5": "Test 5. Assign & single word search", "6": "Test 6. Assign & single search with crossing terms", "7": "Test 7. Assign & repeated search" + }, + "files": { + } + }, + "performance": { + /* In false will prevent running performance tests */ + "run": true, + "tests": { + "test1": { + "alias": "Assign & single search", + "file": "../../../../application/developing/resources/indexing_access_huge.log", + "expectation_ms": 100 + }, + "test2": { + "alias": "Assign & multiple search", + "file": "../../../../application/developing/resources/indexing_access_huge.log", + "expectation_ms": 100 + }, + "test3": { + "alias": "Assign & zero search", + "file": "../../../../application/developing/resources/indexing_access_huge.log", + "expectation_ms": 100 + } } } }, diff --git a/application/apps/rustcore/ts-bindings/spec/session.benchmark.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.benchmark.spec.ts new file mode 100644 index 0000000000..f7897034ef --- /dev/null +++ b/application/apps/rustcore/ts-bindings/spec/session.benchmark.spec.ts @@ -0,0 +1,9 @@ +import { initLogger } from './logger'; +initLogger(); +import { helloWorld } from './common'; + +describe('Hello World', () => { + it('should return "Hello, World!"', () => { + expect(helloWorld()).toEqual('Hello, World!'); + }); +}); \ No newline at end of file diff --git a/application/apps/rustcore/ts-bindings/spec/session.indexes.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.indexes.spec.ts index b86cce4719..9ae2ebf144 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.indexes.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.indexes.spec.ts @@ -6,10 +6,13 @@ import { initLogger } from './logger'; initLogger(); import { Session, Factory } from '../src/api/session'; -import { createSampleFile, finish, runner } from './common'; +import { createSampleFile, finish, performanceReport, setMeasurement, runner } from './common'; import { readConfigurationFile } from './config'; import { Nature, IndexingMode, NatureTypes } from 'platform/types/content'; +import * as os from 'os'; +import * as path from 'path'; + const config = readConfigurationFile().get().tests.indexes; describe('Indexes', function () { @@ -285,4 +288,118 @@ describe('Indexes', function () { }); }); }); + + + + config.performance.run && + Object.keys(config.regular.execute_only).length >= 0 && + Object.keys(config.performance.tests).forEach((alias: string, index: number) => { + const test = (config.performance.tests as any)[alias]; + const testName = `${test.alias}`; + if (test.ignore) { + console.log(`Test "${testName}" has been ignored`); + return; + } + it(testName, function () { + return runner( + { + list: { 1: testName }, + execute_only: [], + files: {}, + }, + 1, + async (logger, done, collector) => { + const measurement = setMeasurement(); + Session.create() + .then((session: Session) => { + // Set provider into debug mode + session.debug(true, testName); + const stream = session.getStream(); + if (stream instanceof Error) { + finish(session, done, stream); + return; + } + const search = session.getSearch(); + if (search instanceof Error) { + finish(session, done, search); + return; + } + const events = session.getEvents(); + if (events instanceof Error) { + finish(session, done, events); + return; + } + let controlSum = 0; + let countMatches = 0; + let read: boolean = false; + let home_dir = (process.env as any)['SH_HOME_DIR']; + stream + .observe( + new Factory.File() + .asText() + .type(Factory.FileType.Text) + .file(`${home_dir}/${test.file}`) + .get().sterilized(), + ) + .catch(finish.bind(null, session, done)); + const updates: number[] = []; + events.IndexedMapUpdated.subscribe((event) => { + event.len > 0 && updates.push(event.len); + }); + events.StreamUpdated.subscribe(async () => { + read = true; + try { + await search.search([ + { + filter: 'HTTP', + flags: { reg: true, word: true, cases: false }, + }, + ]); + let items = await stream.grabIndexed(0, countMatches); + await stream.setIndexingMode(IndexingMode.Breadcrumbs); + finish(session, done); + } catch (err) { + finish( + undefined, + done, + new Error( + `Fail to finish test due error: ${ + err instanceof Error ? err.message : err + }`, + ), + ); + } + }); + events.FileRead.subscribe(() => { + const results = measurement(); + finish( + session, + done, + performanceReport( + testName, + results.ms, + test.expectation_ms, + `${home_dir}/${test.file}`, + ) + ? undefined + : new Error(`${testName} is fail`), + ); + }); + }) + .catch((err: Error) => { + finish( + undefined, + done, + new Error( + `Fail to create session due error: ${ + err instanceof Error ? err.message : err + }`, + ), + ); + }); + }, + ); + }); + }); + }); diff --git a/application/apps/rustcore/ts-bindings/spec/session.observe.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.observe.spec.ts index eb338d67ea..df2fee5d61 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.observe.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.observe.spec.ts @@ -12,6 +12,8 @@ import { IAttachment } from 'platform/types/content'; import { createSampleFile, finish, performanceReport, setMeasurement, runner } from './common'; import { readConfigurationFile } from './config'; import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; const config = readConfigurationFile().get().tests.observe; @@ -795,10 +797,10 @@ describe('Observe', function () { }); config.performance.run && - Object.keys(config.regular.execute_only).length === 0 && + Object.keys(config.regular.execute_only).length > 0 && Object.keys(config.performance.tests).forEach((alias: string, index: number) => { const test = (config.performance.tests as any)[alias]; - const testName = `Performance test #${index + 1} (${test.alias})`; + const testName = `${test.alias}`; if (test.ignore) { console.log(`Test "${testName}" has been ignored`); return; @@ -827,6 +829,7 @@ describe('Observe', function () { finish(session, done, events); return; } + let home_dir = (process.env as any)['SH_HOME_DIR']; switch (test.open_as) { case 'text': stream @@ -834,7 +837,7 @@ describe('Observe', function () { new Factory.File() .asText() .type(Factory.FileType.Text) - .file(test.file) + .file(`${home_dir}/${test.file}`) .get() .sterilized(), ) @@ -845,7 +848,7 @@ describe('Observe', function () { .observe( new Factory.File() .type(Factory.FileType.Binary) - .file(test.file) + .file(`${home_dir}/${test.file}`) .asDlt({ filter_config: undefined, fibex_file_paths: [], @@ -862,7 +865,7 @@ describe('Observe', function () { .observe( new Factory.File() .type(Factory.FileType.PcapNG) - .file(test.file) + .file(`${home_dir}/${test.file}`) .asDlt({ filter_config: undefined, fibex_file_paths: [], @@ -874,6 +877,26 @@ describe('Observe', function () { ) .catch(finish.bind(null, session, done)); break; + case 'startup_measurement': + const tmpobj = createSampleFile( + 5000, + logger, + (i: number) => `some line data: ${i}\n`, + ); + stream + .observe( + new Factory.Stream() + .asText() + .process({ + command: `less ${tmpobj.name}`, + cwd: process.cwd(), + envs: process.env as { [key: string]: string }, + }) + .get() + .sterilized(), + ) + .catch(finish.bind(null, session, done)); + break; default: finish( undefined, @@ -891,7 +914,7 @@ describe('Observe', function () { testName, results.ms, test.expectation_ms, - test.file, + `${home_dir}/${test.file}`, ) ? undefined : new Error(`${testName} is fail`), @@ -913,4 +936,5 @@ describe('Observe', function () { ); }); }); + }); diff --git a/application/apps/rustcore/ts-bindings/spec/session.search.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.search.spec.ts index 59aebea03b..9d3592bc85 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.search.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.search.spec.ts @@ -7,9 +7,12 @@ import { initLogger } from './logger'; initLogger(); import { Session, Factory } from '../src/api/session'; import { IGrabbedElement } from 'platform/types/content'; -import { finish, createSampleFile, runner } from './common'; +import { finish, createSampleFile, performanceReport, setMeasurement, runner } from './common'; import { readConfigurationFile } from './config'; +import * as os from 'os'; +import * as path from 'path'; + const config = readConfigurationFile().get().tests.search; describe('Search', function () { @@ -999,4 +1002,145 @@ describe('Search', function () { }); }); }); + + config.performance.run && + Object.keys(config.regular.execute_only).length > 0 && + Object.keys(config.performance.tests).forEach((alias: string, index: number) => { + const test = (config.performance.tests as any)[alias]; + const testName = `${test.alias}`; + if (test.ignore) { + console.log(`Test "${testName}" has been ignored`); + return; + } + it(testName, function () { + return runner( + { + list: { 1: testName }, + execute_only: [], + files: {}, + }, + 1, + async (logger, done, collector) => { + const measurement = setMeasurement(); + Session.create() + .then((session: Session) => { + // Set provider into debug mode + session.debug(true, testName); + const stream = session.getStream(); + if (stream instanceof Error) { + finish(session, done, stream); + return; + } + const events = session.getEvents(); + if (events instanceof Error) { + finish(session, done, events); + return; + } + const search = session.getSearch(); + if (search instanceof Error) { + finish(session, done, search); + return; + } + let home_dir = (process.env as any)['SH_HOME_DIR']; + switch (index+1) { + case 1: + stream + .observe( + new Factory.File() + .asText() + .type(Factory.FileType.Text) + .file(`${home_dir}/${test.file}`) + .get() + .sterilized(), + ) + .on('processing', () => { + search + .search([ + { + filter: 'http', + flags: { reg: true, word: false, cases: false }, + }, + ]) + .catch(finish.bind(null, session, done)); + }) + .catch(finish.bind(null, session, done)); + break; + case 2: + stream + .observe( + new Factory.File() + .asText() + .type(Factory.FileType.Text) + .file(`${home_dir}/${test.file}`) + .get() + .sterilized(), + ) + .on('processing', () => { + search + .search([ + { + filter: 'http://www.almhuette-raith.at', + flags: { reg: true, word: false, cases: false }, + }, + { + filter: 'com.apple.hiservices-xpcservice', + flags: { reg: true, word: false, cases: false }, + }, + { + filter: 'Google Chrome Helper', + flags: { reg: true, word: false, cases: false }, + }, + ]) + .catch(finish.bind(null, session, done)); + }) + .catch(finish.bind(null, session, done)); + break; + default: + finish( + undefined, + done, + new Error(`Unsupported format: ${test.open_as}`), + ); + return; + } + events.FileRead.subscribe(() => { + const measurement = setMeasurement(); + const search = session.getSearch(); + if (search instanceof Error) { + finish(session, done, search); + return; + } + search.search([]).then((_maches: number) => { + const results = measurement(); + finish( + session, + done, + performanceReport( + testName, + results.ms, + test.expectation_ms, + `${home_dir}/${test.file}`, + ) + ? undefined + : new Error(`${testName} is fail`), + ); + }); + }); + }) + .catch((err: Error) => { + finish( + undefined, + done, + new Error( + `Fail to create session due error: ${ + err instanceof Error ? err.message : err + }`, + ), + ); + }); + }, + ); + }); + }); + }); diff --git a/application/apps/rustcore/ts-bindings/spec/session.stream.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.stream.spec.ts index c588fa4619..9a26f9b88f 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.stream.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.stream.spec.ts @@ -7,7 +7,7 @@ import { initLogger } from './logger'; initLogger(); import { Session, Factory } from '../src/api/session'; import { IGrabbedElement } from 'platform/types/content'; -import { createSampleFile, finish, runner } from './common'; +import { createSampleFile, finish, performanceReport, setMeasurement, runner } from './common'; import { readConfigurationFile } from './config'; import { utils } from 'platform/log'; @@ -577,4 +577,128 @@ if (process.platform === 'win32') { }); }); }); + + config.performance.run && + Object.keys(config.regular.execute_only).length > 0 && + Object.keys(config.performance.tests).forEach((alias: string, index: number) => { + const test = (config.performance.tests as any)[alias]; + const testName = `${test.alias}`; + if (test.ignore) { + console.log(`Test "${testName}" has been ignored`); + return; + } + it(testName, function () { + return runner( + { + list: { 1: testName }, + execute_only: [], + files: {}, + }, + 1, + async (logger, done, collector) => { + const measurement = setMeasurement(); + try { + switch (index + 1) { + case 1: + const tmpobj1 = createSampleFile( + 5000, + logger, + (i: number) => `some line data: ${i}\n` + ); + const session1 = await Session.create(); + session1.debug(true, testName); + const stream1 = session1.getStream(); + if (stream1 instanceof Error) { + throw stream1; + } + await stream1.observe( + new Factory.Stream() + .asText() + .process({ + command: `less ${tmpobj1.name}`, + cwd: process.cwd(), + envs: process.env as { [key: string]: string }, + }) + .get() + .sterilized() + ); + break; + case 2: + const tmpobj2 = createSampleFile( + 5000, + logger, + (i: number) => `some line data: ${i}\n` + ); + const session2 = await Session.create(); + session2.debug(true, testName); + const stream2 = session2.getStream(); + if (stream2 instanceof Error) { + throw stream2; + } + await stream2.observe( + new Factory.Stream() + .asText() + .process({ + command: `less ${tmpobj2.name}`, + cwd: process.cwd(), + envs: process.env as { [key: string]: string }, + }) + .get() + .sterilized() + ); + finish(undefined, done); + break; + case 3: + const results = []; + for (let i = 0; i < 50; i++) { + const file = createSampleFile( + 100, + logger, + (j: number) => `file ${i} line data: ${j}\n` + ); + + const session = await Session.create(); + session.debug(true, `${testName} - session ${i}`); + const stream = session.getStream(); + if (stream instanceof Error) { + throw stream; + } + let result = await stream.observe( + new Factory.Stream() + .asText() + .process({ + command: `less ${file.name}`, + cwd: process.cwd(), + envs: process.env as { [key: string]: string }, + }) + .get() + .sterilized() + ).catch((err) => `File ${i} failed to open: ${err.message}`); + results.push(result); + } + finish(undefined, done); + break; + default: + throw new Error(`Unsupported format: ${test.open_as}`); + } + const results = measurement(); + finish( + undefined, + done, + performanceReport(testName, results.ms, test.expectation_ms) + ? undefined + : new Error(`${testName} is fail`) + ); + } catch (err) { + finish( + undefined, + done, + new Error(`Fail to create session due error: ${err instanceof Error ? err.message : err}`) + ); + } + } + ); + }); + }); + } diff --git a/application/apps/rustcore/ts-bindings/spec/setup_config.sh b/application/apps/rustcore/ts-bindings/spec/setup_config.sh old mode 100644 new mode 100755 index cde78d9070..42c1c4d606 --- a/application/apps/rustcore/ts-bindings/spec/setup_config.sh +++ b/application/apps/rustcore/ts-bindings/spec/setup_config.sh @@ -1 +1,8 @@ -export JASMIN_TEST_CONFIGURATION=put_path_to_config_file \ No newline at end of file +export JASMIN_TEST_CONFIGURATION="./spec/benchmarks.json" +export SH_HOME_DIR="/chipmunk" +if [ "$#" -gt 0 ]; then + export PERFORMANCE_RESULTS_FOLDER="chipmunk_performance_results" + export PERFORMANCE_RESULTS="Benchmark_$1.json" +else + echo "No arguments provided." +fi \ No newline at end of file diff --git a/scripts/elements/bindings.rb b/scripts/elements/bindings.rb index 970b60a9c5..0d47eef14a 100644 --- a/scripts/elements/bindings.rb +++ b/scripts/elements/bindings.rb @@ -14,9 +14,20 @@ module Bindings TARGETS = [DIST, TS_NODE_MODULES, TARGET, DIST_RS, SPEC, TS_BINDINGS_LIB].freeze def self.run_jasmine_spec(spec) + run_benchmarks = ENV['JASMIN_TEST_CONFIGURATION'] && ENV['JASMIN_TEST_CONFIGURATION'].include?('benchmarks.json') ? true : false ENV['ELECTRON_RUN_AS_NODE'] = '1' Shell.chdir(Paths::TS_BINDINGS) do - Shell.sh "#{Paths::JASMINE} spec/build/spec/session.#{spec}.spec.js" + if run_benchmarks + for i in 1..6 do + begin + Shell.sh "#{Paths::JASMINE} spec/build/spec/session.#{spec}.spec.js" + rescue + next + end + end + else + Shell.sh "#{Paths::JASMINE} spec/build/spec/session.#{spec}.spec.js" + end end end end @@ -60,6 +71,7 @@ def self.run_jasmine_spec(spec) errors stream promises + benchmark ] test_specs.each do |spec| desc "run jasmine #{spec}-spec" diff --git a/scripts/tools/run_benchmarks.rb b/scripts/tools/run_benchmarks.rb new file mode 100644 index 0000000000..559dac5ff1 --- /dev/null +++ b/scripts/tools/run_benchmarks.rb @@ -0,0 +1,213 @@ +require 'octokit' +require 'tmpdir' +require 'fileutils' + +REPO_OWNER = 'esrlabs' +REPO_NAME = 'chipmunk' + +RAKE_COMMANDS = [ + 'rake clobber', + 'rake bindings:build', + 'rake bindings:build_spec', + 'rake bindings:test:stream', + 'rake bindings:test:indexes', + 'rake bindings:test:search', + 'rake bindings:test:observe' +] + +SHELL_SCRIPT_PATH = 'application/apps/rustcore/ts-bindings/spec' + +if ARGV.length > 1 + puts "Usage: ruby scripts/tools/run_benchmarks.rb /" + exit(1) +end + +def compute_average_of_benchmarks(result_path) + # Read the JSON data from the file + file_content = File.read(result_path) + data = JSON.parse(file_content) + + # Group the tests by their names and compute the average actual value for each test type + grouped_data = data.group_by { |test| test['name'] } + puts "grouped_data is \n#{grouped_data}" + + averages = grouped_data.map do |name, tests| + passed_tests = tests.select { |test| test['passed'] } + + if passed_tests.any? + average_actual = passed_tests.sum { |test| test['actual'].to_f } / passed_tests.size + average_expected = passed_tests.sum { |test| test['expectation'].to_f } / passed_tests.size + { 'name' => name, 'actual' => average_actual, 'expectation' => average_expected, 'passed' => average_actual <= average_expected } + else + { 'name' => name, 'actual' => 0, 'expectation' => 0, 'passed' => false } + end + end + + puts "Final data is \n#{averages}" + + # Write the resulting JSON back to the file + File.open(result_path, 'w') do |file| + file.write(JSON.pretty_generate(averages)) + end + puts "Average actual values have been written to the file #{result_path}" +end + +# def compute_average_of_benchmarks(result_path) +# # Read the JSON data from the file +# file_content = File.read(result_path) +# data = JSON.parse(file_content) + +# # Group the tests by their names and compute the average actual value for each test type +# grouped_data = data.group_by { |test| test['name'] } +# puts "grouped_data is \n#{grouped_data}" +# averages = grouped_data.map do |name, tests| +# average_actual = tests.sum { |test| test['actual'] if test['passed'] rescue 0} / tests.sum{ |test| test['passed'] ? 1 : 0 rescue 0} +# average_expected = tests.sum { |test| test['expectation'] if test['passed'] rescue 0} / tests.sum{ |test| test['passed'] ? 1 : 0 rescue 0} +# { 'name' => name, 'actual' => average_actual, "expectation": average_expected, 'passed' => average_actual <= average_expected } +# end + +# puts "Final data is \n#{averages}" + +# # Write the resulting JSON back to the file +# File.open(result_path, 'w') do |file| +# file.write(JSON.pretty_generate(averages)) +# end +# puts "Average actual values have been written to the file #{result_path}" +# end + +client = Octokit::Client.new() + +# Fetch the latest releases from the GitHub repository +releases = client.releases("#{REPO_OWNER}/#{REPO_NAME}") + +if !ARGV[0] || (ARGV.length == 1 && ARGV[0].match?(/\A\d+(\.\d+)?\z/)) + NUMBER_OF_RELEASES = ARGV[0].to_i == 0 ? 1 : ARGV[0].to_i + filtered_releases = releases.take(NUMBER_OF_RELEASES) + puts "running benchmarks for last #{NUMBER_OF_RELEASES} releases" +elsif ARGV.length == 1 && ARGV[0].include?('-') + start_tag, end_tag = ARGV[0].split('-') + if start_tag.nil? || end_tag.nil? + puts "Invalid range format. Use " + exit(1) + end + filtered_releases = releases.select do |release| + release.tag_name >= start_tag && release.tag_name <= end_tag + end + puts "running benchmarks for releases #{start_tag} - #{end_tag}" +else + puts "Usage: ruby scripts/tools/run_benchmarks.rb /" + exit(1) +end + +# Iterate over the specified number of releases +filtered_releases.each_with_index do |release, index| + puts "Processing release #{index + 1}: #{release.name}" + + ENV_VARS = { + 'JASMIN_TEST_CONFIGURATION' => './spec/benchmarks.json', + 'PERFORMANCE_RESULTS_FOLDER' => 'chipmunk_performance_results', + 'PERFORMANCE_RESULTS' => "Benchmark_#{release.tag_name}.json", + # 'SH_HOME_DIR' => "/chipmunk" + 'SH_HOME_DIR' => "/Users/sameer.g.srivastava" + } + + # Create a temporary directory for this release + Dir.mktmpdir do |temp_dir| + begin + # Clone the repository into the temporary directory + system("git clone --depth 1 --branch #{release.tag_name} https://github.com/#{REPO_OWNER}/#{REPO_NAME}.git #{temp_dir}") + + # Copy scripts folder to have the test cases available in the cloned repo + FileUtils.cp_r("#{SHELL_SCRIPT_PATH}/.", "#{temp_dir}/#{SHELL_SCRIPT_PATH}/.", verbose: true) + FileUtils.cp_r("scripts/elements/bindings.rb", "#{temp_dir}/scripts/elements/bindings.rb", verbose: true) + + result_path = "#{ENV_VARS['SH_HOME_DIR']}/#{ENV_VARS['PERFORMANCE_RESULTS_FOLDER']}/Benchmark_#{release.tag_name}.json" + + # Change directory to the temporary directory + Dir.chdir(temp_dir) do + # Execute the shell script + ENV_VARS.each do |key, value| + ENV[key] = "#{value}" + end + + system("corepack enable") + system("yarn cache clean") + + if File.exist?("#{SHELL_SCRIPT_PATH}/#{ENV_VARS['JASMIN_TEST_CONFIGURATION'].gsub('./spec/', '')}") + puts "Benchmark.json file available." + else + break + end + + system("printenv") + + if File.exist?(result_path) + FileUtils.rm(result_path, verbose: true) + end + + # Run each Rake command + RAKE_COMMANDS.each do |command| + puts "Running #{command} for tag #{release.name}" + system(command) + end + end + + rescue => e + puts "An error occurred while processing release #{release.tag_name}: #{e.message}" + end + + if File.exist?(result_path) + compute_average_of_benchmarks(result_path) + puts "Benchmark results:" + system("cat #{result_path}") + else + puts "Benchmark results not found at #{result_path}." + end + puts "Completed processing release #{index + 1}: #{release.name}" + + end +end + +# Method to read and parse JSON files +def read_benchmark_data(file_path) + puts "Data in file = #{file_path}\n#{File.read(file_path)}\n:: EOF" + { file_name: File.basename(file_path), data: JSON.parse(File.read(file_path)) } +end + +# Method to collect data from the latest 10 JSON files +def collect_latest_benchmark_data(directory) + Dir.glob("#{directory}/Benchmark*.json").map do |file| + read_benchmark_data(file) + end +end + +# Method to generate graphs for each performance test type +def update_performance_data(data) + # Hash to store data organized by test name + test_data = {} + + # Collect data by test name + data.each do |benchmark| + benchmark[:data].each do |entry| + test_name = entry['name'] + actual_value = entry['actual'] + file_name = benchmark[:file_name] + + if test_data[test_name] + test_data[test_name] << { release: file_name.gsub("Benchmark_","").gsub(".json",""), actual_value: actual_value } + else + test_data[test_name] = [{ release: file_name.gsub("Benchmark_","").gsub(".json",""), actual_value: actual_value }] + end + end + end + + puts ("Test data = #{test_data.to_json}") + File.open("#{ENV_VARS['SH_HOME_DIR']}/#{ENV_VARS['PERFORMANCE_RESULTS_FOLDER']}/data.json", 'w') do |file| + file.write(test_data.to_json) + end + puts "Benchmark data created successfully!" +end + +benchmark_data = collect_latest_benchmark_data("#{ENV_VARS['SH_HOME_DIR']}/#{ENV_VARS['PERFORMANCE_RESULTS_FOLDER']}") +update_performance_data(benchmark_data) + diff --git a/scripts/tools/run_benchmarks1.rb b/scripts/tools/run_benchmarks1.rb new file mode 100644 index 0000000000..e89fabc3c6 --- /dev/null +++ b/scripts/tools/run_benchmarks1.rb @@ -0,0 +1,208 @@ +require 'octokit' +require 'tmpdir' +require 'fileutils' + +ENV['REPO_OWNER'] = 'esrlabs' if !ENV['REPO_OWNER'] +ENV['REPO_NAME'] = 'chipmunk' if !ENV['REPO_NAME'] + +RAKE_COMMANDS = [ + 'rake clobber', + 'rake bindings:build', + 'rake bindings:build_spec', + 'rake bindings:test:stream', + 'rake bindings:test:indexes', + 'rake bindings:test:search', + 'rake bindings:test:observe' +] + +SHELL_SCRIPT_PATH = 'application/apps/rustcore/ts-bindings/spec' + +if ARGV.length != 1 + puts "Usage: ruby scripts/tools/run_benchmarks.rb //PR~" + exit(1) +end + +def compute_average_of_benchmarks(result_path) + # Read the JSON data from the file + file_content = File.read(result_path) + data = JSON.parse(file_content) + + # Group the tests by their names and compute the average actual value for each test type + grouped_data = data.group_by { |test| test['name'] } + puts "grouped_data is \n#{grouped_data}" + + averages = grouped_data.map do |name, tests| + passed_tests = tests.select { |test| test['passed'] } + + if passed_tests.any? + average_actual = passed_tests.sum { |test| test['actual'].to_f } / passed_tests.size + average_expected = passed_tests.sum { |test| test['expectation'].to_f } / passed_tests.size + { 'name' => name, 'actual' => average_actual, 'expectation' => average_expected, 'passed' => average_actual <= average_expected } + else + { 'name' => name, 'actual' => 0, 'expectation' => 0, 'passed' => false } + end + end + + puts "Final data is \n#{averages}" + + # Write the resulting JSON back to the file + File.open(result_path, 'w') do |file| + file.write(JSON.pretty_generate(averages)) + end + puts "Average actual values have been written to the file #{result_path}" +end + +client = Octokit::Client.new() + +if ARGV[0].start_with?('PR~') + pr_number = ARGV[0].split('~').last + # Fetch the pull request by numbers + pull_request = client.pull_request("esrlabs/chipmunk", pr_number) + branch_name = pull_request.head.ref + puts "Running benchmarks for the pull request: #{pull_request.title} (#{branch_name})" +else + # Fetch the latest releases from the GitHub repository + releases = client.releases("#{ENV['REPO_OWNER']}/#{ENV['REPO_NAME']}") + + if !ARGV[0] || (ARGV.length == 1 && ARGV[0].match?(/\A\d+(\.\d+)?\z/)) + NUMBER_OF_RELEASES = ARGV[0].to_i == 0 ? 1 : ARGV[0].to_i + filtered_releases = releases.take(NUMBER_OF_RELEASES) + puts "running benchmarks for last #{NUMBER_OF_RELEASES} releases" + elsif ARGV.length == 1 && ARGV[0].include?('-') + start_tag, end_tag = ARGV[0].split('-') + if start_tag.nil? || end_tag.nil? + puts "Invalid range format. Use " + exit(1) + end + filtered_releases = releases.select do |release| + release.tag_name >= start_tag && release.tag_name <= end_tag + end + puts "running benchmarks for releases #{start_tag} - #{end_tag}" + else + puts "Usage: ruby scripts/tools/run_benchmarks.rb //PR~" + exit(1) + end +end + +ENV_VARS = { + 'JASMIN_TEST_CONFIGURATION' => './spec/benchmarks.json', + 'PERFORMANCE_RESULTS_FOLDER' => 'chipmunk_performance_results', + 'PERFORMANCE_RESULTS' => '', + 'SH_HOME_DIR' => "/chipmunk" + # 'SH_HOME_DIR' => "/Users/sameer.g.srivastava" +} + +def process_release_or_pr(branch_or_tag_name, identifier) + # Create a temporary directory for this release + Dir.mktmpdir do |temp_dir| + begin + # Clone the repository into the temporary directory + system("git clone --depth 1 --branch #{branch_or_tag_name} https://github.com/#{ENV['REPO_OWNER']}/#{ENV['REPO_NAME']}.git #{temp_dir}") + + # Copy scripts folder to have the test cases available in the cloned repo + FileUtils.cp_r("#{SHELL_SCRIPT_PATH}/.", "#{temp_dir}/#{SHELL_SCRIPT_PATH}/.", verbose: true) + FileUtils.cp_r("scripts/elements/bindings.rb", "#{temp_dir}/scripts/elements/bindings.rb", verbose: true) + + result_path = "#{ENV_VARS['SH_HOME_DIR']}/#{ENV_VARS['PERFORMANCE_RESULTS_FOLDER']}/Benchmark_#{identifier}.json" + + # Change directory to the temporary directory + Dir.chdir(temp_dir) do + # Execute the shell script + ENV_VARS.each do |key, value| + ENV[key] = "#{value}" + end + ENV['PERFORMANCE_RESULTS'] = "Benchmark_#{identifier}.json" + system("corepack enable") + system("yarn cache clean") + + if File.exist?("#{SHELL_SCRIPT_PATH}/#{ENV_VARS['JASMIN_TEST_CONFIGURATION'].gsub('./spec/', '')}") + puts "Benchmark.json file available." + else + break + end + + system("printenv") + + if File.exist?(result_path) + FileUtils.rm(result_path, verbose: true) + end + + # Run each Rake command + RAKE_COMMANDS.each do |command| + puts "Running #{command} for #{identifier}" + system(command) + end + end + + rescue => e + puts "An error occurred while processing #{identifier}: #{e.message}" + end + + if File.exist?(result_path) + compute_average_of_benchmarks(result_path) + puts "Benchmark results:" + system("cat #{result_path}") + else + puts "Benchmark results not found at #{result_path}." + end + puts "Completed processing #{identifier}" + + end +end + +if ARGV[0].start_with?('PR~') + process_release_or_pr(branch_name, "PR_#{pr_number}") +else + # Iterate over the specified number of releases + filtered_releases.each_with_index do |release, index| + process_release_or_pr(release.tag_name, release.tag_name) + end +end + +# Method to read and parse JSON files +def read_benchmark_data(file_path) + puts "Data in file = #{file_path}\n#{File.read(file_path)}\n:: EOF" + { file_name: File.basename(file_path), data: JSON.parse(File.read(file_path)) } +end + +# Method to collect data from the latest 10 JSON files +def collect_latest_benchmark_data(directory) + Dir.glob("#{directory}/Benchmark_*.json").reject { |file| File.basename(file).start_with?('Benchmark_PR') }.map do |file| + read_benchmark_data(file) + end +end + +# Method to generate graphs for each performance test type +def update_performance_data(data) + # Hash to store data organized by test name + test_data = {} + + # Collect data by test name + data.each do |benchmark| + benchmark[:data].each do |entry| + test_name = entry['name'] + actual_value = entry['actual'] + file_name = benchmark[:file_name] + + if test_data[test_name] + test_data[test_name] << { release: file_name.gsub("Benchmark_","").gsub(".json",""), actual_value: actual_value } + else + test_data[test_name] = [{ release: file_name.gsub("Benchmark_","").gsub(".json",""), actual_value: actual_value }] + end + end + end + + puts ("Test data = #{test_data.to_json}") + File.open("#{DATA_JSON_PATH}", 'w') do |file| + file.write(test_data.to_json) + end + puts "Benchmark data created successfully!" +end + +benchmark_data = collect_latest_benchmark_data("#{ENV_VARS['SH_HOME_DIR']}/#{ENV_VARS['PERFORMANCE_RESULTS_FOLDER']}") + +# Check if data.json is older than any of the benchmark JSON files +DATA_JSON_PATH = "#{ENV_VARS['SH_HOME_DIR']}/#{ENV_VARS['PERFORMANCE_RESULTS_FOLDER']}/data.json" +if !File.exist?(DATA_JSON_PATH) || benchmark_data.any? { |b| File.mtime(DATA_JSON_PATH) < File.mtime("#{ENV_VARS['SH_HOME_DIR']}/#{ENV_VARS['PERFORMANCE_RESULTS_FOLDER']}/#{b[:file_name]}") } + update_performance_data(benchmark_data) +end