feat(managers/custom): generic manager for json files

docs/usage/configuration-options.md

@@ -777,6 +777,19 @@ Example:
 }
 ```
 
+```json
+{
+  "customManagers": [
+    {
+      "customType": "jsonata",
+      "matchStrings": [
+        "packages.{ \"depName\": package, \"currentValue\": version }"
+      ]
+    }
+  ]
+}
+```
+
 ### datasourceTemplate
 
 If the `datasource` for a dependency is not captured with a named group then it can be defined in config using this field.
@@ -799,18 +812,29 @@ It will be compiled using Handlebars and the regex `groups` result.
 
 ### matchStrings
 
-Each `matchStrings` must be a valid regular expression, optionally with named capture groups.
+Each `matchStrings` must be one of the two:
+
+1. a valid regular expression, optionally with named capture groups (if using `customType=regex`)
+2. a valid, escaped [JSONata](https://docs.jsonata.org/overview.html) query (if using `customType=json`)
 
 Example:
 
-```json
+```json title="matchStrings with a valid regular expression"
 {
   "matchStrings": [
     "ENV .*?_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)\\s"
   ]
 }
 ```
 
+```json title="matchStrings with a valid JSONata query"
+{
+  "matchStrings": [
+    "packages.{ \"depName\": package, \"currentValue\": version }"
+  ]
+}
+```
+
 ### matchStringsStrategy
 
 `matchStringsStrategy` controls behavior when multiple `matchStrings` values are provided.

lib/config/options/index.ts

@@ -2733,18 +2733,17 @@ const options: RenovateOptions[] = [
     description:
       'Custom manager to use. Valid only within a `customManagers` object.',
     type: 'string',
-    allowedValues: ['regex'],
+    allowedValues: ['jsonata', 'regex'],
     parents: ['customManagers'],
     cli: false,
     env: false,
   },
   {
     name: 'matchStrings',
     description:
-      'Regex capture rule to use. Valid only within a `customManagers` object.',
+      'Regex pattern or JSONata query to use. Valid only within a `customManagers` object.',
     type: 'array',
     subType: 'string',
-    format: 'regex',
     parents: ['customManagers'],
     cli: false,
     env: false,

lib/modules/manager/custom/api.ts

@@ -1,7 +1,9 @@
 import type { ManagerApi } from '../types';
+import * as jsonata from './jsonata';
 import * as regex from './regex';
 
 const api = new Map<string, ManagerApi>();
 export default api;
 
 api.set('regex', regex);
+api.set('jsonata', jsonata);

lib/modules/manager/custom/index.spec.ts

@@ -10,6 +10,8 @@ describe('modules/manager/custom/index', () => {
       expect(customManager.isCustomManager('npm')).toBe(false);
       expect(customManager.isCustomManager('regex')).toBe(true);
       expect(customManager.isCustomManager('custom.regex')).toBe(false);
+      expect(customManager.isCustomManager('jsonata')).toBe(true);
+      expect(customManager.isCustomManager('custom.jsonata')).toBe(false);
     });
   });
 });

lib/modules/manager/custom/jsonata/index.spec.ts

@@ -0,0 +1,278 @@
+import { logger } from '../../../../../test/util';
+import type { JsonataExtractConfig } from './types';
+import { defaultConfig, extractPackageFile } from '.';
+
+describe('modules/manager/custom/jsonata/index', () => {
+  it('has default config', () => {
+    expect(defaultConfig).toEqual({
+      pinDigests: false,
+    });
+  });
+
+  it('extracts data when no templates are used', async () => {
+    const json = `
+    {
+    "packages": [
+      {
+        "dep_name": "foo",
+        "package_name": "fii",
+        "current_value": "1.2.3",
+        "current_digest": "1234",
+        "data_source": "nuget",
+        "versioning": "maven",
+        "extract_version": "custom-extract-version",
+        "registry_url": "https://registry.npmjs.org",
+        "dep_type": "dev"
+      }
+      ]
+    }`;
+    const config = {
+      matchStrings: [
+        `packages.{
+            "depName": dep_name,
+            "packageName": package_name,
+            "currentValue": current_value,
+            "currentDigest": current_digest,
+            "datasource": data_source,
+            "versioning": versioning,
+            "extractVersion": extract_version,
+            "registryUrl": registry_url,
+            "depType": dep_type
+        }`,
+      ],
+    };
+    const res = await extractPackageFile(json, 'unused', config);
+
+    expect(res?.deps).toHaveLength(1);
+    expect(res?.deps.filter((dep) => dep.depName === 'foo')).toHaveLength(1);
+    expect(res?.deps.filter((dep) => dep.packageName === 'fii')).toHaveLength(
+      1,
+    );
+    expect(
+      res?.deps.filter((dep) => dep.currentValue === '1.2.3'),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) => dep.currentDigest === '1234'),
+    ).toHaveLength(1);
+    expect(res?.deps.filter((dep) => dep.datasource === 'nuget')).toHaveLength(
+      1,
+    );
+    expect(res?.deps.filter((dep) => dep.versioning === 'maven')).toHaveLength(
+      1,
+    );
+    expect(
+      res?.deps.filter(
+        (dep) => dep.extractVersion === 'custom-extract-version',
+      ),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) =>
+        dep.registryUrls?.includes('https://registry.npmjs.org/'),
+      ),
+    ).toHaveLength(1);
+    expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1);
+  });
+
+  it('applies templates', async () => {
+    const json = `
+    {
+    "packages": [
+      {
+        "dep_name": "foo",
+        "package_name": "fii",
+        "current_value": "1.2.3",
+        "current_digest": "1234",
+        "data_source": "nuget",
+        "versioning": "maven",
+        "extract_version": "custom-extract-version",
+        "registry_url": "https://registry.npmjs.org",
+        "dep_type": "dev"
+      },
+      {
+      }]
+    }`;
+    const config = {
+      matchStrings: [
+        `packages.{
+            "depName": dep_name,
+            "packageName": package_name,
+            "currentValue": current_value,
+            "currentDigest": current_digest,
+            "datasource": data_source,
+            "versioning": versioning,
+            "extractVersion": extract_version,
+            "registryUrl": registry_url,
+            "depType": dep_type
+        }`,
+      ],
+      depNameTemplate:
+        '{{#if depName}}{{depName}}{{else}}default-dep-name{{/if}}',
+      packageNameTemplate:
+        '{{#if packageName}}{{packageName}}{{else}}default-package-name{{/if}}',
+      currentValueTemplate:
+        '{{#if currentValue}}{{currentValue}}{{else}}default-current-value{{/if}}',
+      currentDigestTemplate:
+        '{{#if currentDigest}}{{currentDigest}}{{else}}default-current-digest{{/if}}',
+      datasourceTemplate:
+        '{{#if datasource}}{{datasource}}{{else}}default-datasource{{/if}}',
+      versioningTemplate:
+        '{{#if versioning}}{{versioning}}{{else}}default-versioning{{/if}}',
+      extractVersionTemplate:
+        '{{#if extractVersion}}{{extractVersion}}{{else}}default-extract-version{{/if}}',
+      registryUrlTemplate:
+        '{{#if registryUrl}}{{registryUrl}}{{else}}https://default.registry.url{{/if}}',
+      depTypeTemplate:
+        '{{#if depType}}{{depType}}{{else}}default-dep-type{{/if}}',
+    };
+    const res = await extractPackageFile(json, 'unused', config);
+
+    expect(res?.deps).toHaveLength(2);
+
+    expect(res?.deps.filter((dep) => dep.depName === 'foo')).toHaveLength(1);
+    expect(res?.deps.filter((dep) => dep.packageName === 'fii')).toHaveLength(
+      1,
+    );
+    expect(
+      res?.deps.filter((dep) => dep.currentValue === '1.2.3'),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) => dep.currentDigest === '1234'),
+    ).toHaveLength(1);
+    expect(res?.deps.filter((dep) => dep.datasource === 'nuget')).toHaveLength(
+      1,
+    );
+    expect(res?.deps.filter((dep) => dep.versioning === 'maven')).toHaveLength(
+      1,
+    );
+    expect(
+      res?.deps.filter(
+        (dep) => dep.extractVersion === 'custom-extract-version',
+      ),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) =>
+        dep.registryUrls?.includes('https://registry.npmjs.org/'),
+      ),
+    ).toHaveLength(1);
+    expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1);
+
+    expect(
+      res?.deps.filter((dep) => dep.depName === 'default-dep-name'),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) => dep.packageName === 'default-package-name'),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) => dep.currentValue === 'default-current-value'),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) => dep.currentDigest === 'default-current-digest'),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) => dep.datasource === 'default-datasource'),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) => dep.versioning === 'default-versioning'),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter(
+        (dep) => dep.extractVersion === 'default-extract-version',
+      ),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) =>
+        dep.registryUrls?.includes('https://default.registry.url/'),
+      ),
+    ).toHaveLength(1);
+    expect(
+      res?.deps.filter((dep) => dep.depType === 'default-dep-type'),
+    ).toHaveLength(1);
+  });
+
+  it('returns null when content is not json', async () => {
+    const res = await extractPackageFile(
+      'not-json',
+      'foo-file',
+      {} as JsonataExtractConfig,
+    );
+    expect(res).toBeNull();
+    expect(logger.logger.warn).toHaveBeenCalledWith(
+      expect.anything(),
+      'File is not a valid JSON file.',
+    );
+  });
+
+  it('returns null when no content', async () => {
+    const res = await extractPackageFile(
+      '',
+      'foo-file',
+      {} as JsonataExtractConfig,
+    );
+    expect(res).toBeNull();
+  });
+
+  it('returns null if no dependencies found', async () => {
+    const config = {
+      matchStrings: [
+        'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }',
+      ],
+    };
+    const res = await extractPackageFile('{}', 'unused', config);
+    expect(res).toBeNull();
+  });
+
+  it('returns null if invalid template', async () => {
+    const config = {
+      matchStrings: [`{"depName": "foo"}`],
+      versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver', // invalid template
+    };
+    const res = await extractPackageFile('{}', 'unused', config);
+    expect(res).toBeNull();
+    expect(logger.logger.warn).toHaveBeenCalledWith(
+      expect.anything(),
+      'Error compiling template for JSONata manager',
+    );
+  });
+
+  it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => {
+    const config = {
+      matchStrings: [`{"depName": "foo"}`],
+      registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}',
+    };
+    const res = await extractPackageFile('{}', 'unused', config);
+    expect(res).not.toBeNull();
+    expect(logger.logger.warn).toHaveBeenCalledWith(
+      { value: 'this-is-not-a-valid-url-foo' },
+      'Invalid JSONata manager registryUrl',
+    );
+  });
+
+  it('extracts multiple dependencies with multiple matchStrings', async () => {
+    const config = {
+      matchStrings: [`{"depName": "foo"}`, `{"depName": "bar"}`],
+    };
+    const res = await extractPackageFile('{}', 'unused', config);
+    expect(res?.deps).toHaveLength(2);
+  });
+
+  it('excludes and warns if invalid jsonata query found', async () => {
+    const config = {
+      matchStrings: ['{', `{"depName": "foo"}`, `{"depName": "bar"}`],
+    };
+    const res = await extractPackageFile('{}', 'unused', config);
+    expect(res?.deps).toHaveLength(2);
+    expect(logger.logger.warn).toHaveBeenCalledWith(
+      { err: expect.any(Object) },
+      `Failed to compile JSONata query: {. Excluding it from queries.`,
+    );
+  });
+
+  it('extracts dependency with autoReplaceStringTemplate', async () => {
+    const config = {
+      matchStrings: [`{"depName": "foo"}`],
+      autoReplaceStringTemplate: 'auto-replace-string-template',
+    };
+    const res = await extractPackageFile('{}', 'values.yaml', config);
+    expect(res?.autoReplaceStringTemplate).toBe('auto-replace-string-template');
+  });
+});