From fcaa7ce1e708fec2e8463ae66ffc8e6a2999516c Mon Sep 17 00:00:00 2001 From: Lukas Mittag Date: Wed, 24 Apr 2024 10:04:02 +0200 Subject: [PATCH] Feature: cache array support (#239) * Add support for double quoted arrays * Tests for array support * extend documentation for array support in cache * Write arrays as JSON strings to env variables * Adapt workflow and update license * Add check for type array * Unit tests and fixed regex --- docs/features/PROJECT-CACHE.md | 15 ++++ src/modules/exec.ts | 19 +---- src/modules/stdout-parser.ts | 39 ++++++++++ src/modules/variables.ts | 19 +++-- test/system-test/exec.stest.ts | 9 ++- test/unit/stdout-parser.test.ts | 76 +++++++++++++++++++ .../test-package/test-version/set-cache.py | 5 ++ 7 files changed, 158 insertions(+), 24 deletions(-) create mode 100644 src/modules/stdout-parser.ts create mode 100644 test/unit/stdout-parser.test.ts diff --git a/docs/features/PROJECT-CACHE.md b/docs/features/PROJECT-CACHE.md index 8ada5e45..b8778915 100644 --- a/docs/features/PROJECT-CACHE.md +++ b/docs/features/PROJECT-CACHE.md @@ -40,6 +40,21 @@ echo "myKey=myValue >> VELOCITAS_CACHE" print("myKey=myValue >> VELOCITAS_CACHE") ``` +#### String array support +The cache write supports string array values as well. The elements need to be double quoted and comma separated. + +Examples: + +**Bash** +```bash +echo 'myKey=["myValue", "myValue2"] >> VELOCITAS_CACHE' +``` + +**Python 3** +```python +print('myKey=["myValue", "myValue2"] >> VELOCITAS_CACHE') +``` + ### Reading values To read values from the cache, programs may read the `VELOCITAS_CACHE_DATA` environment variable. It contains a the entire cache data as JSON-string. diff --git a/src/modules/exec.ts b/src/modules/exec.ts index 414751be..f2965ac6 100644 --- a/src/modules/exec.ts +++ b/src/modules/exec.ts @@ -18,26 +18,15 @@ import { join, resolve } from 'node:path'; import { ExecSpec, ProgramSpec } from './component'; import { ProjectCache } from './project-cache'; import { ProjectConfig } from './project-config'; - -const CACHE_OUTPUT_REGEX: RegExp = /(\w+)\s*=\s*(\'.*?\'|\".*?\"|\w+)\s+\>\>\s+VELOCITAS_CACHE/; +import { stdOutParser } from './stdout-parser'; const lineCapturer = (projectCache: ProjectCache, writeStdout: boolean, data: string) => { if (writeStdout) { process.stdout.write(data); } - for (let line of data.toString().split('\n')) { - let lineTrimmed = (line as string).trim(); - - if (lineTrimmed.length === 0) { - continue; - } - const result = CACHE_OUTPUT_REGEX.exec(lineTrimmed); - if (result && result.length > 0) { - const key = result[1]; - const value = result[2].replaceAll("'", '').replaceAll('"', ''); - projectCache.set(key, value); - } - } + data.toString() + .split('\n') + .forEach((value) => stdOutParser(projectCache, value)); }; export function setSpawnImplementation(func: (command: string, args: string | string[], options: any) => IPty) { diff --git a/src/modules/stdout-parser.ts b/src/modules/stdout-parser.ts new file mode 100644 index 00000000..d4751fba --- /dev/null +++ b/src/modules/stdout-parser.ts @@ -0,0 +1,39 @@ +// Copyright (c) 2024 Contributors to the Eclipse Foundation +// +// This program and the accompanying materials are made available under the +// terms of the Apache License, Version 2.0 which is available at +// https://www.apache.org/licenses/LICENSE-2.0. +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +import { ProjectCache } from './project-cache'; + +const CACHE_OUTPUT_REGEX: RegExp = + /(\w+)\s*=\s*(\[((\'(\/*\w+)*\'\s*,\s*|(\"(\/*\w+)*\"\s*,\s*)|(\/*\w+)*\s*,\s*|\'(\/*\w+)*\'(?=\])|\"(\/*\w+)*\"(?=\])|(\/*\w+)*(?=\]))*\])|(\'.*?\'|\".*?\"|\w+))\s+\>\>\s+VELOCITAS_CACHE/; + +export function stdOutParser(projectCache: ProjectCache, line: string) { + let lineTrimmed = (line as string).trim(); + + if (lineTrimmed.length === 0) { + return; + } + const match = CACHE_OUTPUT_REGEX.exec(line); + if (match && match.length > 0) { + const [ignored, key, value] = match; + const cleanedValue = value.replace(/['"]/g, ''); + if (cleanedValue.startsWith('[') && cleanedValue.endsWith(']')) { + const arrayPart = cleanedValue.substring(1, cleanedValue.length - 1); + const array = arrayPart.split(','); + const trimmedArray = array.map((str) => str.trim()); + projectCache.set(key, trimmedArray); + } else { + projectCache.set(key, cleanedValue); + } + } +} diff --git a/src/modules/variables.ts b/src/modules/variables.ts index 47c95988..bf2b07d8 100644 --- a/src/modules/variables.ts +++ b/src/modules/variables.ts @@ -71,9 +71,9 @@ export class VariableCollection { for (const [key, value] of this._variables.entries()) { const transformedKey = key.replaceAll('.', '_'); - + const transformedValue = Array.isArray(value) ? JSON.stringify(value) : value; Object.assign(envVars, { - [transformedKey]: value, + [transformedKey]: transformedValue, }); } @@ -196,17 +196,20 @@ function verifyGivenVariables( for (const componentExposedVariable of variableDefinitions) { const configuredValue = configuredVars.get(componentExposedVariable.name); + const errorMsg = `'${componentExposedVariable.name}' has wrong type! Expected ${ + componentExposedVariable.type + } but got ${typeof configuredValue}`; if (!configuredValue) { if (componentExposedVariable.default === undefined) { missingVars.push(componentExposedVariable); } } else { - if (typeof configuredValue !== componentExposedVariable.type) { - wronglyTypedVars.push( - `'${componentExposedVariable.name}' has wrong type! Expected ${ - componentExposedVariable.type - } but got ${typeof configuredValue}`, - ); + if (componentExposedVariable.type === 'array') { + if (!Array.isArray(configuredValue)) { + wronglyTypedVars.push(errorMsg); + } + } else if (typeof configuredValue !== componentExposedVariable.type) { + wronglyTypedVars.push(errorMsg); } configuredVars.delete(componentExposedVariable.name); } diff --git a/test/system-test/exec.stest.ts b/test/system-test/exec.stest.ts index f1970e6d..61237f53 100644 --- a/test/system-test/exec.stest.ts +++ b/test/system-test/exec.stest.ts @@ -70,7 +70,6 @@ describe('CLI command', () => { it('should be able to let programs get cache values', () => { const result = spawnSync(VELOCITAS_PROCESS, ['exec', 'test-component', 'get-cache'], { encoding: DEFAULT_BUFFER_ENCODING }); - expect(result.stdout).to.contain('my_cache_key'); expect(result.stdout).to.contain('my_cache_value'); expect(result.stdout).to.contain('foo'); @@ -81,6 +80,14 @@ describe('CLI command', () => { expect(result.stdout).to.contain('random'); expect(result.stdout).to.not.contain('var'); expect(result.stdout).to.not.contain('asdc'); + expect(result.stdout).to.contain('arr1'); + expect(result.stdout).to.contain("['/path/test1', '/path/test2']"); + expect(result.stdout).to.contain('arr2'); + expect(result.stdout).to.contain("['/path/test1', '/path/test2']"); + expect(result.stdout).to.contain('arr3'); + expect(result.stdout).to.contain("['/path/test1', '/path/test2']"); + expect(result.stdout).to.contain('arr4'); + expect(result.stdout).to.contain("['/path/test1', '/path/test2']"); }); it('should be able to run programs which read from stdin', () => { diff --git a/test/unit/stdout-parser.test.ts b/test/unit/stdout-parser.test.ts new file mode 100644 index 00000000..866841b7 --- /dev/null +++ b/test/unit/stdout-parser.test.ts @@ -0,0 +1,76 @@ +// Copyright (c) 2024 Contributors to the Eclipse Foundation +// +// This program and the accompanying materials are made available under the +// terms of the Apache License, Version 2.0 which is available at +// https://www.apache.org/licenses/LICENSE-2.0. +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +import { expect } from 'chai'; +import { ProjectCache } from '../../src/modules/project-cache'; +import { stdOutParser } from '../../src/modules/stdout-parser'; +import { CliFileSystem, MockFileSystem, MockFileSystemObj } from '../../src/utils/fs-bridge'; +import { getCacheData } from '../helpers/cache'; + +describe('stdOutParser - module', () => { + + before(() => { + const projectCacheDir = ProjectCache.getCacheDir(); + const mockFilesystem: MockFileSystemObj = { + [`${projectCacheDir}/cache.json`]: '{}', + }; + CliFileSystem.setImpl(new MockFileSystem(mockFilesystem)); + }); + + describe('valid inputs', () => { + const matches = new Map([ + ['test_1 = "/this/is/path/one" >> VELOCITAS_CACHE', '/this/is/path/one'], + ["test_2 = '/this/is/path/one' >> VELOCITAS_CACHE", '/this/is/path/one'], + ['test_3 = test3 >> VELOCITAS_CACHE', 'test3'], + ['test_4 = ["/this/is/path/one", "/this/is/path/two"] >> VELOCITAS_CACHE', ['/this/is/path/one', '/this/is/path/two']], + ['test_5 = ["/this/is/path/one","/this/is/path/two"] >> VELOCITAS_CACHE', ['/this/is/path/one', '/this/is/path/two']], + ['test_6 = ["/this/is/path/one", "/this/is/path/two"] >> VELOCITAS_CACHE', ['/this/is/path/one', '/this/is/path/two']], + ['test_7 =["/this/is/path/one", "/this/is/path/two"] >> VELOCITAS_CACHE', ['/this/is/path/one', '/this/is/path/two']], + ['test_8=["/this/is/path/one", "/this/is/path/two"] >> VELOCITAS_CACHE', ['/this/is/path/one', '/this/is/path/two']], + ["test_9 = ['/this/is/path/one', /this/is/path/two] >> VELOCITAS_CACHE", ['/this/is/path/one', '/this/is/path/two']], + ['test_10 = [/this/is/path/one, "/this/is/path/two"] >> VELOCITAS_CACHE', ['/this/is/path/one', '/this/is/path/two']], + ['test_11 = [/this/is/path/one, /this/is/path/two] >> VELOCITAS_CACHE', ['/this/is/path/one', '/this/is/path/two']], + ]); + + it('should match and set project cache correct', () => { + matches.forEach((result, element) => { + let id = element.split('=')[0].trim(); + const projectCache = ProjectCache.read(); + stdOutParser(projectCache, element); + expect(JSON.stringify(projectCache.get(id))).to.equal(JSON.stringify(result)); + }); + }); + }); + + describe('invalid inputs', () => { + const noMatches = new Map([ + ['test_1 = "/this/is/path/one >> VELOCITAS_CACHE', {}], + ["test_2 = '/this/is/path/one >> VELOCITAS_CACHE", {}], + ['test_3 = /this/is/path/one"this/test >> VELOCITAS_CACHE', {}], + ['test_4 = ["/this/is/path/one", /this/is/path/two"]', {}], + ['test_5 = ["/this/is/path/one""/this/is/path/two"] >> VELOCITAS_CACHE', {}], + ['"test_6" = ["/this/is/path/one", "/this/is/path/two"] >> VELOCITAS_CACHE', {}], + ['test_7 = [/this/is/path/one""/this/is/path/two"] >> VELOCITAS_CACHE', {}], + ["test_8 = ['/this/is/path/one''/this/is/path/two'] >> VELOCITAS_CACHE", {}], + ]); + + it('should not match for wrong inputs', async () => { + noMatches.forEach((result, element) => { + const projectCache = ProjectCache.read(); + stdOutParser(projectCache, element); + expect(JSON.stringify(getCacheData())).to.equal(JSON.stringify(result)); + }); + }); + }); +}); diff --git a/testbench/test-exec/packages/test-package/test-version/set-cache.py b/testbench/test-exec/packages/test-package/test-version/set-cache.py index eb7a374e..665f1b41 100644 --- a/testbench/test-exec/packages/test-package/test-version/set-cache.py +++ b/testbench/test-exec/packages/test-package/test-version/set-cache.py @@ -16,4 +16,9 @@ print("foo = bar >> VELOCITAS_CACHE") print("x=y >> VELOCITAS_CACHE") print("0123='random' >> VELOCITAS_CACHE") +print('arr1=["/path/test1", "/path/test2"] >> VELOCITAS_CACHE') +print("arr2=['/path/test1', '/path/test2'] >> VELOCITAS_CACHE") +print("arr3=['/path/test1', /path/test2] >> VELOCITAS_CACHE") +print('arr4=["/path/test1","/path/test2"] >> VELOCITAS_CACHE') +print('arr5=["/path/test1" ,"/path/test2"] >> VELOCITAS_CACHE') print("var=\"asdc' >> VELOCITAS_CACHE")