From 54c79272c1b1876823acaa13f0ee6a48ec501e93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=A4der?= Date: Fri, 12 Jul 2024 09:08:28 +0200 Subject: [PATCH] Add logging to `download:plugins` script (#13905) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Attempted mitigation for #13902 - do not eat exceptions and properly log errors - reduces request rate to 3/sec. Contributed on behalf of STMicroelectronics Signed-off-by: Thomas Mäder --- .github/workflows/ci-cd.yml | 2 +- dev-packages/cli/src/download-plugins.ts | 1 + .../ovsx-client/src/ovsx-api-filter.ts | 5 +- .../ovsx-client/src/ovsx-http-client.ts | 24 +----- dev-packages/ovsx-client/src/ovsx-types.ts | 1 - .../request/src/node-request-service.ts | 14 +++- .../src/browser/vsx-extensions-model.ts | 76 +++++++++++-------- .../vsx-language-quick-pick-service.ts | 70 ++++++++--------- 8 files changed, 99 insertions(+), 94 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 06c97f26cbaa0..e563ca511eb20 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -86,7 +86,7 @@ jobs: if: runner.os == 'Linux' shell: bash run: | - yarn -s download:plugins + yarn -s download:plugins --rate-limit 3 - name: Build shell: bash diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts index 7aa0abdc5d640..dee69530e72c8 100644 --- a/dev-packages/cli/src/download-plugins.ts +++ b/dev-packages/cli/src/download-plugins.ts @@ -143,6 +143,7 @@ export default async function downloadPlugins(ovsxClient: OVSXClient, requestSer failures.push(`No download url for extension pack ${id} (${version})`); } } catch (err) { + console.error(err); failures.push(err.message); } })); diff --git a/dev-packages/ovsx-client/src/ovsx-api-filter.ts b/dev-packages/ovsx-client/src/ovsx-api-filter.ts index 33b1f197e2b1c..a470a4ee3f989 100644 --- a/dev-packages/ovsx-client/src/ovsx-api-filter.ts +++ b/dev-packages/ovsx-client/src/ovsx-api-filter.ts @@ -69,9 +69,10 @@ export class OVSXApiFilterImpl implements OVSXApiFilter { let offset = 0; let loop = true; while (loop) { - const queryOptions = { + const queryOptions: VSXQueryOptions = { ...query, - offset + offset, + size: 5 // there is a great chance that the newest version will work }; const results = await this.client.query(queryOptions); const compatibleExtension = this.getLatestCompatibleExtension(results.extensions); diff --git a/dev-packages/ovsx-client/src/ovsx-http-client.ts b/dev-packages/ovsx-client/src/ovsx-http-client.ts index f59f9615d7ff6..a810680711f30 100644 --- a/dev-packages/ovsx-client/src/ovsx-http-client.ts +++ b/dev-packages/ovsx-client/src/ovsx-http-client.ts @@ -34,28 +34,12 @@ export class OVSXHttpClient implements OVSXClient { protected requestService: RequestService ) { } - async search(searchOptions?: VSXSearchOptions): Promise { - try { - return await this.requestJson(this.buildUrl('api/-/search', searchOptions)); - } catch (err) { - return { - error: err?.message || String(err), - offset: -1, - extensions: [] - }; - } + search(searchOptions?: VSXSearchOptions): Promise { + return this.requestJson(this.buildUrl('api/-/search', searchOptions)); } - async query(queryOptions?: VSXQueryOptions): Promise { - try { - return await this.requestJson(this.buildUrl('api/v2/-/query', queryOptions)); - } catch (error) { - return { - offset: 0, - totalSize: 0, - extensions: [] - }; - } + query(queryOptions?: VSXQueryOptions): Promise { + return this.requestJson(this.buildUrl('api/v2/-/query', queryOptions)); } protected async requestJson(url: string): Promise { diff --git a/dev-packages/ovsx-client/src/ovsx-types.ts b/dev-packages/ovsx-client/src/ovsx-types.ts index 223e3ef5bea6f..ec747ebf91e8e 100644 --- a/dev-packages/ovsx-client/src/ovsx-types.ts +++ b/dev-packages/ovsx-client/src/ovsx-types.ts @@ -105,7 +105,6 @@ export interface VSXSearchOptions { * Should be aligned with https://github.com/eclipse/openvsx/blob/e8f64fe145fc05d2de1469735d50a7a90e400bc4/server/src/main/java/org/eclipse/openvsx/json/SearchResultJson.java */ export interface VSXSearchResult { - error?: string; offset: number; extensions: VSXSearchEntry[]; } diff --git a/dev-packages/request/src/node-request-service.ts b/dev-packages/request/src/node-request-service.ts index 4126647f6451a..e31a022bc1ef6 100644 --- a/dev-packages/request/src/node-request-service.ts +++ b/dev-packages/request/src/node-request-service.ts @@ -136,11 +136,19 @@ export class NodeRequestService implements RequestService { }); }); - stream.on('error', reject); + stream.on('error', err => { + reject(err); + }); } }); - req.on('error', reject); + req.on('error', err => { + reject(err); + }); + + req.on('timeout', () => { + reject('timeout'); + }); if (options.timeout) { req.setTimeout(options.timeout); @@ -153,7 +161,7 @@ export class NodeRequestService implements RequestService { req.end(); token?.onCancellationRequested(() => { - req.abort(); + req.destroy(); reject(); }); }); diff --git a/packages/vsx-registry/src/browser/vsx-extensions-model.ts b/packages/vsx-registry/src/browser/vsx-extensions-model.ts index 12649d8a5092e..b050cf0d54d76 100644 --- a/packages/vsx-registry/src/browser/vsx-extensions-model.ts +++ b/packages/vsx-registry/src/browser/vsx-extensions-model.ts @@ -225,49 +225,59 @@ export class VSXExtensionsModel { } const client = await this.clientProvider(); const filter = await this.vsxApiFilter(); - const result = await client.search(param); - this._searchError = result.error; - if (token.isCancellationRequested) { - return; - } - for (const data of result.extensions) { - const id = data.namespace.toLowerCase() + '.' + data.name.toLowerCase(); - const allVersions = filter.getLatestCompatibleVersion(data); - if (!allVersions) { - continue; + try { + const result = await client.search(param); + + if (token.isCancellationRequested) { + return; } - if (this.preferences.get('extensions.onlyShowVerifiedExtensions')) { - this.fetchVerifiedStatus(id, client, allVersions).then(verified => { - this.doChange(() => { - this.addExtensions(data, id, allVersions, !!verified); - return Promise.resolve(); + for (const data of result.extensions) { + const id = data.namespace.toLowerCase() + '.' + data.name.toLowerCase(); + const allVersions = filter.getLatestCompatibleVersion(data); + if (!allVersions) { + continue; + } + if (this.preferences.get('extensions.onlyShowVerifiedExtensions')) { + this.fetchVerifiedStatus(id, client, allVersions).then(verified => { + this.doChange(() => { + this.addExtensions(data, id, allVersions, !!verified); + return Promise.resolve(); + }); }); - }); - } else { - this.addExtensions(data, id, allVersions); - this.fetchVerifiedStatus(id, client, allVersions).then(verified => { - this.doChange(() => { - let extension = this.getExtension(id); - extension = this.setExtension(id); - extension.update(Object.assign({ - verified: verified - })); - return Promise.resolve(); + } else { + this.addExtensions(data, id, allVersions); + this.fetchVerifiedStatus(id, client, allVersions).then(verified => { + this.doChange(() => { + let extension = this.getExtension(id); + extension = this.setExtension(id); + extension.update(Object.assign({ + verified: verified + })); + return Promise.resolve(); + }); }); - }); + } } + } catch (error) { + this._searchError = error?.message || String(error); } + }, token); } protected async fetchVerifiedStatus(id: string, client: OVSXClient, allVersions: VSXAllVersions): Promise { - const res = await client.query({ extensionId: id, extensionVersion: allVersions.version, includeAllVersions: true }); - const extension = res.extensions?.[0]; - let verified = extension?.verified; - if (!verified && extension?.publishedBy.loginName === 'open-vsx') { - verified = true; + try { + const res = await client.query({ extensionId: id, extensionVersion: allVersions.version, includeAllVersions: true }); + const extension = res.extensions?.[0]; + let verified = extension?.verified; + if (!verified && extension?.publishedBy.loginName === 'open-vsx') { + verified = true; + } + return verified; + } catch (error) { + console.error(error); + return false; } - return verified; } protected addExtensions(data: VSXSearchEntry, id: string, allVersions: VSXAllVersions, verified?: boolean): void { diff --git a/packages/vsx-registry/src/browser/vsx-language-quick-pick-service.ts b/packages/vsx-registry/src/browser/vsx-language-quick-pick-service.ts index 6fc219c9247a2..717a35d1fe048 100644 --- a/packages/vsx-registry/src/browser/vsx-language-quick-pick-service.ts +++ b/packages/vsx-registry/src/browser/vsx-language-quick-pick-service.ts @@ -42,47 +42,49 @@ export class VSXLanguageQuickPickService extends LanguageQuickPickService { protected override async getAvailableLanguages(): Promise { const client = await this.clientProvider(); - const searchResult = await client.search({ - category: 'Language Packs', - sortBy: 'downloadCount', - sortOrder: 'desc', - size: 20 - }); - if (searchResult.error) { - throw new Error('Error while loading available languages: ' + searchResult.error); - } + try { + const searchResult = await client.search({ + category: 'Language Packs', + sortBy: 'downloadCount', + sortOrder: 'desc', + size: 20 + }); - const extensionLanguages = await Promise.all( - searchResult.extensions.map(async extension => ({ - extension, - languages: await this.loadExtensionLanguages(extension) - })) - ); + const extensionLanguages = await Promise.all( + searchResult.extensions.map(async extension => ({ + extension, + languages: await this.loadExtensionLanguages(extension) + })) + ); - const languages = new Map(); + const languages = new Map(); - for (const extension of extensionLanguages) { - for (const localizationContribution of extension.languages) { - if (!languages.has(localizationContribution.languageId)) { - languages.set(localizationContribution.languageId, { - ...this.createLanguageQuickPickItem(localizationContribution), - execute: async () => { - const progress = await this.messageService.showProgress({ - text: nls.localizeByDefault('Installing {0} language support...', - localizationContribution.localizedLanguageName ?? localizationContribution.languageName ?? localizationContribution.languageId), - }); - try { - const extensionUri = VSCodeExtensionUri.fromId(`${extension.extension.namespace}.${extension.extension.name}`).toString(); - await this.pluginServer.deploy(extensionUri); - } finally { - progress.cancel(); + for (const extension of extensionLanguages) { + for (const localizationContribution of extension.languages) { + if (!languages.has(localizationContribution.languageId)) { + languages.set(localizationContribution.languageId, { + ...this.createLanguageQuickPickItem(localizationContribution), + execute: async () => { + const progress = await this.messageService.showProgress({ + text: nls.localizeByDefault('Installing {0} language support...', + localizationContribution.localizedLanguageName ?? localizationContribution.languageName ?? localizationContribution.languageId), + }); + try { + const extensionUri = VSCodeExtensionUri.fromId(`${extension.extension.namespace}.${extension.extension.name}`).toString(); + await this.pluginServer.deploy(extensionUri); + } finally { + progress.cancel(); + } } - } - }); + }); + } } } + return Array.from(languages.values()); + } catch (error) { + console.error(error); + return []; } - return Array.from(languages.values()); } protected async loadExtensionLanguages(extension: VSXSearchEntry): Promise {