From 274e93700d8a8f8b26bff015dcbef37a801ffb63 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Fri, 22 Nov 2024 20:38:39 +0530 Subject: [PATCH 1/9] populate latest tag --- lib/modules/datasource/cpan/index.spec.ts | 12 +++++++++++- lib/modules/datasource/cpan/index.ts | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/modules/datasource/cpan/index.spec.ts b/lib/modules/datasource/cpan/index.spec.ts index c4727a1281d990..2007aa3474170c 100644 --- a/lib/modules/datasource/cpan/index.spec.ts +++ b/lib/modules/datasource/cpan/index.spec.ts @@ -63,7 +63,16 @@ describe('modules/datasource/cpan/index', () => { (body) => body.query.filtered.filter.and[0].term['module.name'] === 'Plack', ) - .reply(200, Fixtures.get('Plack.json')); + .reply(200, Fixtures.get('Plack.json')) + .get('/v1/module/Plack') + .reply(200, { + module: [ + { + name: 'Plack', + version: '1.0048', + }, + ], + }); const res = await getPkgReleases({ datasource: CpanDatasource.id, packageName: 'Plack', @@ -86,6 +95,7 @@ describe('modules/datasource/cpan/index', () => { releaseTimestamp: '2020-11-30T00:21:36.000Z', version: '1.0048', }); + expect(res?.tags?.latest).toBe('1.0048'); }); }); }); diff --git a/lib/modules/datasource/cpan/index.ts b/lib/modules/datasource/cpan/index.ts index 85436a1d925463..ceabe73687aae6 100644 --- a/lib/modules/datasource/cpan/index.ts +++ b/lib/modules/datasource/cpan/index.ts @@ -107,8 +107,24 @@ export class CpanDatasource extends Datasource { changelogUrl: `https://metacpan.org/dist/${latestDistribution}/changes`, homepage: `https://metacpan.org/pod/${packageName}`, }; + + const latestVersionUrl = joinUrlParts( + registryUrl, + 'v1/module/', + packageName, + ); + const latestVersionRes = ( + await this.http.getJson(latestVersionUrl) + ).body; + const latestVersion = latestVersionRes.module[0].version; + + if (latestVersion) { + result.tags ??= {}; + result.tags.latest ??= latestVersion; + } } } + return result; } } From bdc18375c0f8af4ab4b44f553daa3e2abf70b528 Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Fri, 22 Nov 2024 20:42:30 +0530 Subject: [PATCH 2/9] Apply suggestions from code review --- lib/modules/datasource/cpan/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/modules/datasource/cpan/index.ts b/lib/modules/datasource/cpan/index.ts index ceabe73687aae6..d20dbb0e39fd51 100644 --- a/lib/modules/datasource/cpan/index.ts +++ b/lib/modules/datasource/cpan/index.ts @@ -124,7 +124,6 @@ export class CpanDatasource extends Datasource { } } } - return result; } } From 20a6168bc530ed9b7a7c93f908c568ad3a96aaf6 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Fri, 22 Nov 2024 21:47:33 +0530 Subject: [PATCH 3/9] apply suggestions --- lib/modules/datasource/cpan/index.ts | 29 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/modules/datasource/cpan/index.ts b/lib/modules/datasource/cpan/index.ts index d20dbb0e39fd51..57be7a4975050a 100644 --- a/lib/modules/datasource/cpan/index.ts +++ b/lib/modules/datasource/cpan/index.ts @@ -1,3 +1,4 @@ +import { logger } from '../../../logger'; import { cache } from '../../../util/cache/package/decorator'; import { joinUrlParts } from '../../../util/url'; import * as perlVersioning from '../../versioning/perl'; @@ -108,19 +109,23 @@ export class CpanDatasource extends Datasource { homepage: `https://metacpan.org/pod/${packageName}`, }; - const latestVersionUrl = joinUrlParts( - registryUrl, - 'v1/module/', - packageName, - ); - const latestVersionRes = ( - await this.http.getJson(latestVersionUrl) - ).body; - const latestVersion = latestVersionRes.module[0].version; + try { + const latestVersionUrl = joinUrlParts( + registryUrl, + 'v1/module/', + packageName, + ); + const latestVersionRes = ( + await this.http.getJson(latestVersionUrl) + ).body; + const latestVersion = latestVersionRes?.module?.[0]?.version; - if (latestVersion) { - result.tags ??= {}; - result.tags.latest ??= latestVersion; + if (latestVersion) { + result.tags ??= {}; + result.tags.latest ??= latestVersion; + } + } catch { + logger.debug(`Error while fetching latest tag for ${packageName}.`); } } } From d2136a413d5d5221425d075225b819215e9f9f27 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sun, 24 Nov 2024 17:45:55 +0530 Subject: [PATCH 4/9] use zod schema --- .../datasource/cpan/__fixtures__/Plack.json | 33 ++++++++++ lib/modules/datasource/cpan/index.spec.ts | 11 +--- lib/modules/datasource/cpan/schema.ts | 64 +++++++++++++++++++ lib/modules/datasource/types.ts | 1 + 4 files changed, 99 insertions(+), 10 deletions(-) create mode 100644 lib/modules/datasource/cpan/schema.ts diff --git a/lib/modules/datasource/cpan/__fixtures__/Plack.json b/lib/modules/datasource/cpan/__fixtures__/Plack.json index 9fb33a17fd6236..444eb672d7d75e 100644 --- a/lib/modules/datasource/cpan/__fixtures__/Plack.json +++ b/lib/modules/datasource/cpan/__fixtures__/Plack.json @@ -24,6 +24,7 @@ "version" : "1.0048" } ], + "status": "latest", "date" : "2020-11-30T00:21:36", "maturity" : "released", "distribution" : "Plack", @@ -47,6 +48,7 @@ "name" : "Plack" } ], + "status": "cpan", "date" : "2018-02-10T09:25:30", "maturity" : "released" }, @@ -63,6 +65,7 @@ "name" : "Plack" } ], + "status": "cpan", "date" : "2018-02-10T07:52:31", "deprecated" : false }, @@ -84,6 +87,7 @@ "name" : "Plack" } ], + "status": "cpan", "date" : "2017-12-31T20:42:50", "distribution" : "Plack", "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0045.tar.gz" @@ -107,6 +111,7 @@ "version" : "1.0044" } ], + "status": "cpan", "date" : "2017-04-27T17:48:20", "maturity" : "released", "deprecated" : false @@ -134,6 +139,7 @@ "version" : "1.0043" } ], + "status": "cpan", "date" : "2017-02-22T03:02:05", "maturity" : "released", "distribution" : "Plack", @@ -158,6 +164,7 @@ "version" : "1.0042" } ], + "status": "cpan", "date" : "2016-09-29T05:38:42", "deprecated" : false } @@ -172,6 +179,7 @@ "version" : "1.0041" } ], + "status": "cpan", "date" : "2016-09-25T21:25:47", "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0041.tar.gz", "distribution" : "Plack" @@ -195,6 +203,7 @@ "name" : "Plack" } ], + "status": "cpan", "date" : "2016-04-01T16:58:21", "maturity" : "developer" }, @@ -217,6 +226,7 @@ "name" : "Plack" } ], + "status": "cpan", "date" : "2015-12-06T11:29:40", "distribution" : "Plack", "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0039.tar.gz" @@ -227,6 +237,29 @@ "sort" : [ 1449401380000 ] + }, + { + "_type" : "file", + "_source" : { + "deprecated" : false, + "maturity" : "released", + "module" : [ + { + "version" : "1.0039", + "name" : "Plack" + } + ], + "date" : "2015-12-06T11:29:40", + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0039.tar.gz" + }, + "status": "invalid", + "_index" : "cpan_v1_01", + "_id" : "Y7WlIYOZjk3rh9O05F3UZE6WwGo", + "_score" : null, + "sort" : [ + 1449401380000 + ] } ], "max_score" : null diff --git a/lib/modules/datasource/cpan/index.spec.ts b/lib/modules/datasource/cpan/index.spec.ts index 2007aa3474170c..231e5bd316ab79 100644 --- a/lib/modules/datasource/cpan/index.spec.ts +++ b/lib/modules/datasource/cpan/index.spec.ts @@ -63,16 +63,7 @@ describe('modules/datasource/cpan/index', () => { (body) => body.query.filtered.filter.and[0].term['module.name'] === 'Plack', ) - .reply(200, Fixtures.get('Plack.json')) - .get('/v1/module/Plack') - .reply(200, { - module: [ - { - name: 'Plack', - version: '1.0048', - }, - ], - }); + .reply(200, Fixtures.get('Plack.json')); const res = await getPkgReleases({ datasource: CpanDatasource.id, packageName: 'Plack', diff --git a/lib/modules/datasource/cpan/schema.ts b/lib/modules/datasource/cpan/schema.ts new file mode 100644 index 00000000000000..726b2a5a02db8b --- /dev/null +++ b/lib/modules/datasource/cpan/schema.ts @@ -0,0 +1,64 @@ +import { z } from 'zod'; +import { LooseArray } from '../../../util/schema-utils'; +import type { Release } from '../types'; + +/** + * https://fastapi.metacpan.org/v1/file/_mapping + */ +const MetaCpanApiFileSchema = z + .object({ + module: LooseArray( + z.object({ + name: z.string(), + version: z.string(), + }), + ), + distribution: z.string(), + date: z.string(), + deprecated: z.boolean(), + maturity: z.string(), + status: z.union([ + z.literal('backpan'), + z.literal('cpan'), + z.literal('latest'), + ]), + }) + .transform( + ({ + module, + distribution, + date, + deprecated, + maturity, + status, + }): (Release & { distribution: string }) | undefined => { + return { + version: module[0].version, + distribution, + isDeprecated: deprecated, + isStable: maturity === 'released', + releaseTimestamp: date, + isLatest: status === 'latest', + }; + }, + ) + .catch(undefined); +/** + * https://github.com/metacpan/metacpan-api/blob/master/docs/API-docs.md#available-fields + */ +export const MetaCpanApiFileSearchResponse = z + .object({ + hits: z.object({ + hits: LooseArray( + z.object({ + _source: MetaCpanApiFileSchema, + }), + ), + }), + }) + .transform((data): (Release & { distribution: string })[] => { + // Extract all hits and filter out ones where _source transformed to undefined + return data.hits.hits + .map((hit) => hit._source) + .filter((source) => source !== undefined); + }); diff --git a/lib/modules/datasource/types.ts b/lib/modules/datasource/types.ts index d4e8ed0b3287e2..60ef43e2e1c2fa 100644 --- a/lib/modules/datasource/types.ts +++ b/lib/modules/datasource/types.ts @@ -69,6 +69,7 @@ export interface Release { sourceUrl?: string | undefined; sourceDirectory?: string; currentAge?: string; + isLatest?: boolean; } export interface ReleaseResult { From b70e6357bd46e7db3b5d33d86280cbe864fbee2a Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sun, 24 Nov 2024 17:46:21 +0530 Subject: [PATCH 5/9] use single api call --- lib/modules/datasource/cpan/index.ts | 88 ++++++++++------------------ 1 file changed, 30 insertions(+), 58 deletions(-) diff --git a/lib/modules/datasource/cpan/index.ts b/lib/modules/datasource/cpan/index.ts index 57be7a4975050a..570abf7b7b61eb 100644 --- a/lib/modules/datasource/cpan/index.ts +++ b/lib/modules/datasource/cpan/index.ts @@ -1,10 +1,9 @@ -import { logger } from '../../../logger'; import { cache } from '../../../util/cache/package/decorator'; import { joinUrlParts } from '../../../util/url'; import * as perlVersioning from '../../versioning/perl'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; -import type { MetaCpanApiFile, MetaCpanApiFileSearchResult } from './types'; +import { MetaCpanApiFileSearchResponse } from './schema'; export class CpanDatasource extends Datasource { static readonly id = 'cpan'; @@ -39,7 +38,7 @@ export class CpanDatasource extends Datasource { let result: ReleaseResult | null = null; const searchUrl = joinUrlParts(registryUrl, 'v1/file/_search'); - let hits: MetaCpanApiFile[] | null = null; + let releases: (Release & { distribution: string })[] | null = null; try { const body = { query: { @@ -61,74 +60,47 @@ export class CpanDatasource extends Datasource { 'date', 'deprecated', 'maturity', + 'status', ], sort: [{ date: 'desc' }], }; - const res = await this.http.postJson( - searchUrl, - { body }, - ); - hits = res.body?.hits?.hits?.map(({ _source }) => _source); + + releases = ( + await this.http.postJson( + searchUrl, + { body }, + MetaCpanApiFileSearchResponse, + ) + ).body; } catch (err) { this.handleGenericErrors(err); } let latestDistribution: string | null = null; - if (hits) { - const releases: Release[] = []; - for (const hit of hits) { - const { - module, - distribution, - date: releaseTimestamp, - deprecated: isDeprecated, - maturity, - } = hit; - const version = module.find( - ({ name }) => name === packageName, - )?.version; - if (version) { - // https://metacpan.org/pod/CPAN::DistnameInfo#maturity - const isStable = maturity === 'released'; - releases.push({ - isDeprecated, - isStable, - releaseTimestamp, - version, - }); - - if (!latestDistribution) { - latestDistribution = distribution; - } + let latestVersion: string | null = null; + if (releases) { + for (const release of releases) { + if (!latestDistribution) { + latestDistribution = release.distribution; + } + if (release.isLatest) { + latestVersion = release.version; } } - if (releases.length > 0 && latestDistribution) { - result = { - releases, - changelogUrl: `https://metacpan.org/dist/${latestDistribution}/changes`, - homepage: `https://metacpan.org/pod/${packageName}`, - }; - - try { - const latestVersionUrl = joinUrlParts( - registryUrl, - 'v1/module/', - packageName, - ); - const latestVersionRes = ( - await this.http.getJson(latestVersionUrl) - ).body; - const latestVersion = latestVersionRes?.module?.[0]?.version; + } + if (releases.length > 0 && latestDistribution) { + result = { + releases, + changelogUrl: `https://metacpan.org/dist/${latestDistribution}/changes`, + homepage: `https://metacpan.org/pod/${packageName}`, + }; - if (latestVersion) { - result.tags ??= {}; - result.tags.latest ??= latestVersion; - } - } catch { - logger.debug(`Error while fetching latest tag for ${packageName}.`); - } + if (latestVersion) { + result.tags ??= {}; + result.tags.latest = latestVersion; } } + return result; } } From 996d404d4c9dd2143e11aca45862a114008db833 Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Sun, 24 Nov 2024 17:56:18 +0530 Subject: [PATCH 6/9] Update lib/modules/datasource/cpan/__fixtures__/Plack.json --- lib/modules/datasource/cpan/__fixtures__/Plack.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/datasource/cpan/__fixtures__/Plack.json b/lib/modules/datasource/cpan/__fixtures__/Plack.json index 444eb672d7d75e..7d402d1db68eab 100644 --- a/lib/modules/datasource/cpan/__fixtures__/Plack.json +++ b/lib/modules/datasource/cpan/__fixtures__/Plack.json @@ -245,7 +245,7 @@ "maturity" : "released", "module" : [ { - "version" : "1.0039", + "version" : "1.0038", "name" : "Plack" } ], From 971b76ae0cfb7a264d2f83b48ae4d4a5a73c472f Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Sun, 24 Nov 2024 17:58:13 +0530 Subject: [PATCH 7/9] Update lib/modules/datasource/cpan/index.ts --- lib/modules/datasource/cpan/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/datasource/cpan/index.ts b/lib/modules/datasource/cpan/index.ts index 570abf7b7b61eb..f34c3a258703ab 100644 --- a/lib/modules/datasource/cpan/index.ts +++ b/lib/modules/datasource/cpan/index.ts @@ -83,7 +83,7 @@ export class CpanDatasource extends Datasource { if (!latestDistribution) { latestDistribution = release.distribution; } - if (release.isLatest) { + if (!latestVersion && release.isLatest) { latestVersion = release.version; } } From e27766b99aaab8b389d4fed26d066d5f70e4e702 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sun, 24 Nov 2024 19:59:35 +0530 Subject: [PATCH 8/9] types: CpanRelease interface --- lib/modules/datasource/cpan/schema.ts | 6 +++--- lib/modules/datasource/cpan/types.ts | 25 +++---------------------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/lib/modules/datasource/cpan/schema.ts b/lib/modules/datasource/cpan/schema.ts index 726b2a5a02db8b..8e997143ba3849 100644 --- a/lib/modules/datasource/cpan/schema.ts +++ b/lib/modules/datasource/cpan/schema.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import { LooseArray } from '../../../util/schema-utils'; -import type { Release } from '../types'; +import type { CpanRelease } from './types'; /** * https://fastapi.metacpan.org/v1/file/_mapping @@ -31,7 +31,7 @@ const MetaCpanApiFileSchema = z deprecated, maturity, status, - }): (Release & { distribution: string }) | undefined => { + }): CpanRelease | undefined => { return { version: module[0].version, distribution, @@ -56,7 +56,7 @@ export const MetaCpanApiFileSearchResponse = z ), }), }) - .transform((data): (Release & { distribution: string })[] => { + .transform((data): CpanRelease[] => { // Extract all hits and filter out ones where _source transformed to undefined return data.hits.hits .map((hit) => hit._source) diff --git a/lib/modules/datasource/cpan/types.ts b/lib/modules/datasource/cpan/types.ts index e6279b5b865da8..0d9cef2408b9a2 100644 --- a/lib/modules/datasource/cpan/types.ts +++ b/lib/modules/datasource/cpan/types.ts @@ -1,24 +1,5 @@ -/** - * https://fastapi.metacpan.org/v1/file/_mapping - */ -export interface MetaCpanApiFile { - module: { - name: string; - version?: string; - }[]; - distribution: string; - date: string; - deprecated: boolean; - maturity: string; -} +import type { Release } from '../types'; -/** - * https://github.com/metacpan/metacpan-api/blob/master/docs/API-docs.md#available-fields - */ -export interface MetaCpanApiFileSearchResult { - hits: { - hits: { - _source: MetaCpanApiFile; - }[]; - }; +export interface CpanRelease extends Release { + distribution: string; } From f2c6998d108519b365fdb87a69bc9d4bd1007df6 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sun, 24 Nov 2024 20:23:08 +0530 Subject: [PATCH 9/9] refactor types --- lib/modules/datasource/cpan/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/cpan/index.ts b/lib/modules/datasource/cpan/index.ts index f34c3a258703ab..435c48764f5907 100644 --- a/lib/modules/datasource/cpan/index.ts +++ b/lib/modules/datasource/cpan/index.ts @@ -2,8 +2,9 @@ import { cache } from '../../../util/cache/package/decorator'; import { joinUrlParts } from '../../../util/url'; import * as perlVersioning from '../../versioning/perl'; import { Datasource } from '../datasource'; -import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; +import type { GetReleasesConfig, ReleaseResult } from '../types'; import { MetaCpanApiFileSearchResponse } from './schema'; +import type { CpanRelease } from './types'; export class CpanDatasource extends Datasource { static readonly id = 'cpan'; @@ -38,7 +39,7 @@ export class CpanDatasource extends Datasource { let result: ReleaseResult | null = null; const searchUrl = joinUrlParts(registryUrl, 'v1/file/_search'); - let releases: (Release & { distribution: string })[] | null = null; + let releases: CpanRelease[] | null = null; try { const body = { query: {