diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 9a00259cbb5ec0..102ebd68bc5526 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -777,6 +777,19 @@ Example: } ``` +```json +{ + "customManagers": [ + { + "customType": "jsonata", + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] + } + ] +} +``` + ### datasourceTemplate If the `datasource` for a dependency is not captured with a named group then it can be defined in config using this field. @@ -799,11 +812,14 @@ It will be compiled using Handlebars and the regex `groups` result. ### matchStrings -Each `matchStrings` must be a valid regular expression, optionally with named capture groups. +Each `matchStrings` must be one of the two: + +1. a valid regular expression, optionally with named capture groups (if using `customType=regex`) +2. a valid, escaped [JSONata](https://docs.jsonata.org/overview.html) query (if using `customType=json`) Example: -```json +```json title="matchStrings with a valid regular expression" { "matchStrings": [ "ENV .*?_VERSION=(?.*) # (?.*?)/(?.*?)\\s" @@ -811,6 +827,14 @@ Example: } ``` +```json title="matchStrings with a valid JSONata query" +{ + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] +} +``` + ### matchStringsStrategy `matchStringsStrategy` controls behavior when multiple `matchStrings` values are provided. diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 712dc338a3b93f..1285df402cbcb3 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2733,7 +2733,7 @@ const options: RenovateOptions[] = [ description: 'Custom manager to use. Valid only within a `customManagers` object.', type: 'string', - allowedValues: ['regex'], + allowedValues: ['jsonata', 'regex'], parents: ['customManagers'], cli: false, env: false, @@ -2741,10 +2741,9 @@ const options: RenovateOptions[] = [ { name: 'matchStrings', description: - 'Regex capture rule to use. Valid only within a `customManagers` object.', + 'Regex pattern or JSONata query to use. Valid only within a `customManagers` object.', type: 'array', subType: 'string', - format: 'regex', parents: ['customManagers'], cli: false, env: false, diff --git a/lib/modules/manager/custom/api.ts b/lib/modules/manager/custom/api.ts index de5e051ca72ca1..f7dc64aaef3cc6 100644 --- a/lib/modules/manager/custom/api.ts +++ b/lib/modules/manager/custom/api.ts @@ -1,7 +1,9 @@ import type { ManagerApi } from '../types'; +import * as jsonata from './jsonata'; import * as regex from './regex'; const api = new Map(); export default api; api.set('regex', regex); +api.set('jsonata', jsonata); diff --git a/lib/modules/manager/custom/index.spec.ts b/lib/modules/manager/custom/index.spec.ts index 54e922bd1ec517..61a6466dbcd447 100644 --- a/lib/modules/manager/custom/index.spec.ts +++ b/lib/modules/manager/custom/index.spec.ts @@ -10,6 +10,8 @@ describe('modules/manager/custom/index', () => { expect(customManager.isCustomManager('npm')).toBe(false); expect(customManager.isCustomManager('regex')).toBe(true); expect(customManager.isCustomManager('custom.regex')).toBe(false); + expect(customManager.isCustomManager('jsonata')).toBe(true); + expect(customManager.isCustomManager('custom.jsonata')).toBe(false); }); }); }); diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts new file mode 100644 index 00000000000000..226e8b1969eba2 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -0,0 +1,278 @@ +import { logger } from '../../../../../test/util'; +import type { JsonataExtractConfig } from './types'; +import { defaultConfig, extractPackageFile } from '.'; + +describe('modules/manager/custom/jsonata/index', () => { + it('has default config', () => { + expect(defaultConfig).toEqual({ + pinDigests: false, + }); + }); + + it('extracts data when no templates are used', async () => { + const json = ` + { + "packages": [ + { + "dep_name": "foo", + "package_name": "fii", + "current_value": "1.2.3", + "current_digest": "1234", + "data_source": "nuget", + "versioning": "maven", + "extract_version": "custom-extract-version", + "registry_url": "https://registry.npmjs.org", + "dep_type": "dev" + } + ] + }`; + const config = { + matchStrings: [ + `packages.{ + "depName": dep_name, + "packageName": package_name, + "currentValue": current_value, + "currentDigest": current_digest, + "datasource": data_source, + "versioning": versioning, + "extractVersion": extract_version, + "registryUrl": registry_url, + "depType": dep_type + }`, + ], + }; + const res = await extractPackageFile(json, 'unused', config); + + expect(res?.deps).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.depName === 'foo')).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.packageName === 'fii')).toHaveLength( + 1, + ); + expect( + res?.deps.filter((dep) => dep.currentValue === '1.2.3'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.currentDigest === '1234'), + ).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.datasource === 'nuget')).toHaveLength( + 1, + ); + expect(res?.deps.filter((dep) => dep.versioning === 'maven')).toHaveLength( + 1, + ); + expect( + res?.deps.filter( + (dep) => dep.extractVersion === 'custom-extract-version', + ), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => + dep.registryUrls?.includes('https://registry.npmjs.org/'), + ), + ).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); + }); + + it('applies templates', async () => { + const json = ` + { + "packages": [ + { + "dep_name": "foo", + "package_name": "fii", + "current_value": "1.2.3", + "current_digest": "1234", + "data_source": "nuget", + "versioning": "maven", + "extract_version": "custom-extract-version", + "registry_url": "https://registry.npmjs.org", + "dep_type": "dev" + }, + { + }] + }`; + const config = { + matchStrings: [ + `packages.{ + "depName": dep_name, + "packageName": package_name, + "currentValue": current_value, + "currentDigest": current_digest, + "datasource": data_source, + "versioning": versioning, + "extractVersion": extract_version, + "registryUrl": registry_url, + "depType": dep_type + }`, + ], + depNameTemplate: + '{{#if depName}}{{depName}}{{else}}default-dep-name{{/if}}', + packageNameTemplate: + '{{#if packageName}}{{packageName}}{{else}}default-package-name{{/if}}', + currentValueTemplate: + '{{#if currentValue}}{{currentValue}}{{else}}default-current-value{{/if}}', + currentDigestTemplate: + '{{#if currentDigest}}{{currentDigest}}{{else}}default-current-digest{{/if}}', + datasourceTemplate: + '{{#if datasource}}{{datasource}}{{else}}default-datasource{{/if}}', + versioningTemplate: + '{{#if versioning}}{{versioning}}{{else}}default-versioning{{/if}}', + extractVersionTemplate: + '{{#if extractVersion}}{{extractVersion}}{{else}}default-extract-version{{/if}}', + registryUrlTemplate: + '{{#if registryUrl}}{{registryUrl}}{{else}}https://default.registry.url{{/if}}', + depTypeTemplate: + '{{#if depType}}{{depType}}{{else}}default-dep-type{{/if}}', + }; + const res = await extractPackageFile(json, 'unused', config); + + expect(res?.deps).toHaveLength(2); + + expect(res?.deps.filter((dep) => dep.depName === 'foo')).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.packageName === 'fii')).toHaveLength( + 1, + ); + expect( + res?.deps.filter((dep) => dep.currentValue === '1.2.3'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.currentDigest === '1234'), + ).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.datasource === 'nuget')).toHaveLength( + 1, + ); + expect(res?.deps.filter((dep) => dep.versioning === 'maven')).toHaveLength( + 1, + ); + expect( + res?.deps.filter( + (dep) => dep.extractVersion === 'custom-extract-version', + ), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => + dep.registryUrls?.includes('https://registry.npmjs.org/'), + ), + ).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); + + expect( + res?.deps.filter((dep) => dep.depName === 'default-dep-name'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.packageName === 'default-package-name'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.currentValue === 'default-current-value'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.currentDigest === 'default-current-digest'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.datasource === 'default-datasource'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.versioning === 'default-versioning'), + ).toHaveLength(1); + expect( + res?.deps.filter( + (dep) => dep.extractVersion === 'default-extract-version', + ), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => + dep.registryUrls?.includes('https://default.registry.url/'), + ), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.depType === 'default-dep-type'), + ).toHaveLength(1); + }); + + it('returns null when content is not json', async () => { + const res = await extractPackageFile( + 'not-json', + 'foo-file', + {} as JsonataExtractConfig, + ); + expect(res).toBeNull(); + expect(logger.logger.warn).toHaveBeenCalledWith( + expect.anything(), + 'File is not a valid JSON file.', + ); + }); + + it('returns null when no content', async () => { + const res = await extractPackageFile( + '', + 'foo-file', + {} as JsonataExtractConfig, + ); + expect(res).toBeNull(); + }); + + it('returns null if no dependencies found', async () => { + const config = { + matchStrings: [ + 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', + ], + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).toBeNull(); + }); + + it('returns null if invalid template', async () => { + const config = { + matchStrings: [`{"depName": "foo"}`], + versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver', // invalid template + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).toBeNull(); + expect(logger.logger.warn).toHaveBeenCalledWith( + expect.anything(), + 'Error compiling template for JSONata manager', + ); + }); + + it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => { + const config = { + matchStrings: [`{"depName": "foo"}`], + registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}', + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).not.toBeNull(); + expect(logger.logger.warn).toHaveBeenCalledWith( + { value: 'this-is-not-a-valid-url-foo' }, + 'Invalid JSONata manager registryUrl', + ); + }); + + it('extracts multiple dependencies with multiple matchStrings', async () => { + const config = { + matchStrings: [`{"depName": "foo"}`, `{"depName": "bar"}`], + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res?.deps).toHaveLength(2); + }); + + it('excludes and warns if invalid jsonata query found', async () => { + const config = { + matchStrings: ['{', `{"depName": "foo"}`, `{"depName": "bar"}`], + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res?.deps).toHaveLength(2); + expect(logger.logger.warn).toHaveBeenCalledWith( + { err: expect.any(Object) }, + `Failed to compile JSONata query: {. Excluding it from queries.`, + ); + }); + + it('extracts dependency with autoReplaceStringTemplate', async () => { + const config = { + matchStrings: [`{"depName": "foo"}`], + autoReplaceStringTemplate: 'auto-replace-string-template', + }; + const res = await extractPackageFile('{}', 'values.yaml', config); + expect(res?.autoReplaceStringTemplate).toBe('auto-replace-string-template'); + }); +}); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts new file mode 100644 index 00000000000000..c36d0d5905c9b5 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -0,0 +1,58 @@ +import is from '@sindresorhus/is'; +import type { Category } from '../../../../constants'; +import { logger } from '../../../../logger'; +import { parseJson } from '../../../../util/common'; +import type { PackageFileContent } from '../../types'; +import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; +import { handleMatching, validMatchFields } from './utils'; + +export const categories: Category[] = ['custom']; + +export const defaultConfig = { + pinDigests: false, +}; +export const supportedDatasources = ['*']; +export const displayName = 'JSONata'; + +export async function extractPackageFile( + content: string, + packageFile: string, + config: JsonataExtractConfig, +): Promise { + let json; + try { + json = parseJson(content, packageFile); + } catch (err) { + logger.warn( + { err, fileName: packageFile }, + 'File is not a valid JSON file.', + ); + return null; + } + + if (is.nullOrUndefined(json)) { + return null; + } + + const deps = await handleMatching(json, packageFile, config); + if (!deps.length) { + return null; + } + + const res: PackageFileContent & JSONataManagerTemplates = { + deps, + matchStrings: config.matchStrings, + }; + // copy over templates for autoreplace + for (const field of validMatchFields.map( + (f) => `${f}Template` as keyof JSONataManagerTemplates, + )) { + if (config[field]) { + res[field] = config[field]; + } + } + if (config.autoReplaceStringTemplate) { + res.autoReplaceStringTemplate = config.autoReplaceStringTemplate; + } + return res; +} diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md new file mode 100644 index 00000000000000..a907e0227737f4 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -0,0 +1,195 @@ +With `customManagers` using `JSONata` queries you can configure Renovate so it finds dependencies in JSON files, that are not detected by its other built-in package managers. + +Renovate uses the `jsonata` package to process the `json` file content. Read about the [jsonata query language](https://docs.jsonata.org/overview.html) in their readme. + +The JSONata manager is unique in Renovate in because: + +- It is configurable via [JSONata](https://jsonata.org/) queries +- It can extract any `datasource` +- By using the `customManagers` config, you can create multiple "JSONata managers" the same repository + +### Required Fields + +The first two required fields are `fileMatch` and `matchStrings`: + +- `fileMatch` works the same as any manager +- `matchStrings` is a `JSONata` custom manager concept and is used for configuring a jsonata queries + +#### Information that Renovate needs about the dependency + +Before Renovate can look up a dependency and decide about updates, it must have this info about each dependency: + +| Info type | Required | Notes | Docs | +| :--------------------------------------------------- | :------- | :-------------------------------------------------------- | :----------------------------------------------------------------------------- | +| Name of the dependency | Yes | | | +| `datasource` | Yes | Example datasources: npm, Docker, GitHub tags, and so on. | [Supported datasources](../../datasource/index.md#supported-datasources) | +| Version scheme to use. Defaults to `semver-coerced`. | Yes | You may set another version scheme, like `pep440`. | [Supported versioning schemes](../../versioning/index.md#supported-versioning) | + +#### Required fields to be present in the resulting structure returned by the jsonata query + +You must: + +- Capture the `currentValue` of the dependency +- Capture the `depName` or `packageName`. Or use a template field: `depNameTemplate` and `packageNameTemplate` +- Capture the `datasource`, or a use `datasourceTemplate` config field + +#### Optional fields you can include in the resulting structure + +You may use any of these items: + +- `depType`, or a use `depTypeTemplate` config field +- `versioning`, or a use `versioningTemplate` config field. If neither are present, Renovate defaults to `semver-coerced` +- `extractVersion`, or use an `extractVersionTemplate` config field +- `currentDigest` +- `registryUrl`, or a use `registryUrlTemplate` config field. If it's a valid URL, it will be converted to the `registryUrls` field as a single-length array +- `indentation`. It must be either empty, or whitespace only (otherwise `indentation` will be reset to an empty string) + +### Usage + +To configure it, use the following syntax: + +```javascript +{ + "customManagers": [ + { + "type": "jsonata", + "fileMatch": [""], + "matchStrings": [''], + ... + } + ] +} +``` + +Where `` is a [JSONata](https://docs.jsonata.org/overview.html) query that transform the contents into a JSON object with the following schema: + +```json5 +{ + depName: '', + packageName: '', // fallback to depName + currentValue: '', + currentDigest: '', // optional + datasource: '', + versioning: '', // optional + extractVersion: '', // optional + registryUrl: '', // optional + depType: '', // optional +} +``` + +To be effective with the JSONata manager, you should understand jsonata queries. But enough examples may compensate for lack of experience. + +#### Example queries + +Below are some example queries for the generic JSON manager. +You can also use the [JSONata test website](https://try.jsonata.org) to experiment with queries. + +```json title="Dependencies spread in different nodes, and we want to limit the extraction to a particular node" +{ + "production": [ + { + "version": "1.2.3", + "package": "foo" + } + ], + "development": [ + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +production.{ "depName": package, "currentValue": version } +``` + +```json title="Dependencies spread in different nodes, and we want to extract all of them as if they were in the same node" +{ + "production": [ + { + "version": "1.2.3", + "package": "foo" + } + ], + "development": [ + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +*.{ "depName": package, "currentValue": version } +``` + +```json title="The dependency name is in a JSON node name and the version is in a child leaf to that node" +{ + "foo": { + "version": "1.2.3" + }, + "bar": { + "version": "4.5.6" + } +} +``` + +Query: + +``` +$each(function($v, $n) { { "depName": $n, "currentValue": $v.version } }) +``` + +```json title="The name of the dependency and the version are both value nodes of the same parent node" +{ + "packages": [ + { + "version": "1.2.3", + "package": "foo" + }, + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +packages.{ "depName": package, "currentValue": version } +``` + +```json title="The name of the dependency and the version are in the same string" +{ + "packages": ["foo@1.2.3", "bar@4.5.6"] +} +``` + +Query: + +``` +$map($map(packages, function ($v) { $split($v, "@") }), function ($v) { { "depName": $v[0], "currentVersion": $v[1] } }) +``` + +```json title="JSONata manager config to extract deps from package.json file in the renovate repository" +{ + "customType": "jsonata", + "fileMatch": ["package.json"], + "matchStrings": [ + "$each(dependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"dependencies\"}})", + "$each(devDependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"devDependencies\"}})", + "$each(optionalDependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"optionalDependencies\"}})", + "{ \"depName\": \"pnpm\", \"currentValue\": $substring(packageManager, 5), \"depType\": \"packageManager\"}" + ], + "datasourceTemplate": "npm" +} +``` diff --git a/lib/modules/manager/custom/jsonata/types.ts b/lib/modules/manager/custom/jsonata/types.ts new file mode 100644 index 00000000000000..eb1da9e30042c1 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/types.ts @@ -0,0 +1,25 @@ +import type { ExtractConfig } from '../../types'; + +export interface JSONataManagerTemplates { + depNameTemplate?: string; + packageNameTemplate?: string; + datasourceTemplate?: string; + versioningTemplate?: string; + depTypeTemplate?: string; + currentValueTemplate?: string; + currentDigestTemplate?: string; + extractVersionTemplate?: string; + registryUrlTemplate?: string; +} + +export interface JSONataManagerConfig extends JSONataManagerTemplates { + matchStrings: string[]; + autoReplaceStringTemplate?: string; +} + +export interface JsonataExtractConfig + extends ExtractConfig, + JSONataManagerTemplates { + autoReplaceStringTemplate?: string; + matchStrings: string[]; +} diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts new file mode 100644 index 00000000000000..eef1548b290b61 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -0,0 +1,102 @@ +import { URL } from 'url'; +import is from '@sindresorhus/is'; +import jsonata from 'jsonata'; +import { logger } from '../../../../logger'; +import * as template from '../../../../util/template'; +import type { PackageDependency } from '../../types'; +import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; + +export const validMatchFields = [ + 'depName', + 'packageName', + 'currentValue', + 'currentDigest', + 'datasource', + 'versioning', + 'extractVersion', + 'registryUrl', + 'depType', +] as const; + +type ValidMatchFields = (typeof validMatchFields)[number]; + +export async function handleMatching( + json: unknown, + packageFile: string, + config: JsonataExtractConfig, +): Promise { + // Pre-compile all JSONata expressions once + const compiledExpressions = config.matchStrings + .map((query) => { + try { + return jsonata(query); + } catch (err) { + logger.warn( + { err }, + `Failed to compile JSONata query: ${query}. Excluding it from queries.`, + ); + return null; + } + }) + .filter((expr) => expr !== null); + + // Execute all expressions in parallel + const results = await Promise.all( + compiledExpressions.map(async (expr) => { + const result = (await expr.evaluate(json)) ?? []; + return is.array(result) ? result : [result]; + }), + ); + + // Flatten results and create dependencies + return results + .flat() + .map((queryResult) => { + return createDependency(queryResult as Record, config); + }) + .filter((dep) => dep !== null); +} + +export function createDependency( + queryResult: Record, + config: JsonataExtractConfig, +): PackageDependency | null { + const dependency: PackageDependency = {}; + + function updateDependency(field: ValidMatchFields, value: string): void { + switch (field) { + case 'registryUrl': + // check if URL is valid and pack inside an array + try { + const url = new URL(value).toString(); + dependency.registryUrls = [url]; + } catch { + logger.warn({ value }, 'Invalid JSONata manager registryUrl'); + } + break; + default: + dependency[field] = value; + break; + } + } + + for (const field of validMatchFields) { + const fieldTemplate = `${field}Template` as keyof JSONataManagerTemplates; + const tmpl = config[fieldTemplate]; + if (tmpl) { + try { + const compiled = template.compile(tmpl, queryResult, false); + updateDependency(field, compiled); + } catch { + logger.warn( + { template: tmpl }, + 'Error compiling template for JSONata manager', + ); + return null; + } + } else if (queryResult[field]) { + updateDependency(field, queryResult[field]); + } + } + return dependency; +} diff --git a/lib/modules/manager/custom/types.ts b/lib/modules/manager/custom/types.ts index de387685f80717..c7edda609f9e35 100644 --- a/lib/modules/manager/custom/types.ts +++ b/lib/modules/manager/custom/types.ts @@ -1,8 +1,11 @@ +import type { JSONataManagerConfig } from './jsonata/types'; import type { RegexManagerConfig } from './regex/types'; -export interface CustomExtractConfig extends Partial {} +export interface CustomExtractConfig + extends Partial, + Partial {} -export type CustomManagerName = 'regex'; +export type CustomManagerName = 'jsonata' | 'regex'; export interface CustomManager extends Partial { customType: CustomManagerName;