diff --git a/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES b/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES index cbb7a61739..46e65c57ec 100644 --- a/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES +++ b/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES @@ -1,9 +1,11 @@ +.openapi-generator-ignore Apis/ImportApi.md Apis/ManagementApi.md Apis/OrganizationApi.md Apis/RepositoryApi.md Models/ApprovalTool.md Models/Import.md +Models/ImportJobListV2.md Models/ImportRequest.md Models/ImportRequest_github.md Models/ImportRequest_github_pullRequest.md @@ -15,5 +17,7 @@ Models/Organization.md Models/OrganizationList.md Models/Repository.md Models/RepositoryList.md +Models/findAllImports_200_response.md +Models/findAllImports_500_response.md Models/ping_200_response.md README.md diff --git a/plugins/bulk-import-backend/api-docs/Apis/ImportApi.md b/plugins/bulk-import-backend/api-docs/Apis/ImportApi.md index 2ed4c68506..07cf0ce910 100644 --- a/plugins/bulk-import-backend/api-docs/Apis/ImportApi.md +++ b/plugins/bulk-import-backend/api-docs/Apis/ImportApi.md @@ -64,7 +64,7 @@ null (empty response body) # **findAllImports** -> List findAllImports(pagePerIntegration, sizePerIntegration, search) +> findAllImports_200_response findAllImports(api-version, pagePerIntegration, sizePerIntegration, page, size, search) Fetch Import Jobs @@ -72,13 +72,16 @@ Fetch Import Jobs |Name | Type | Description | Notes | |------------- | ------------- | ------------- | -------------| -| **pagePerIntegration** | **Integer**| the page number for each Integration | [optional] [default to 1] | -| **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | -| **search** | **String**| returns only Imports that contain the search string, by repository name | [optional] [default to null] | +| **api-version** | **String**| API version. ## Changelog ### v1 (default) Initial version #### Deprecations * GET /imports * Deprecation of 'pagePerIntegration' and 'sizePerIntegration' query parameters and introduction of new 'page' and 'size' parameters * 'page' takes precedence over 'pagePerIntegration' if both are passed * 'size' takes precedence over 'sizePerIntegration' if both are passed ### v2 #### Breaking changes * GET /imports * Query parameters: * 'pagePerIntegration' is ignored in favor of 'page' * 'sizePerIntegration' is ignored in favor of 'size' * Response structure changed to include pagination info: instead of returning a simple list of Imports, the response is now an object containing the following fields: * 'imports': the list of Imports * 'page': the page requested * 'size': the requested number of Imports requested per page * 'totalCount': the total count of Imports | [optional] [default to v1] [enum: v1, v2] | +| **pagePerIntegration** | **Integer**| the page number for each Integration. **Deprecated**. Use the 'page' query parameter instead. | [optional] [default to 1] | +| **sizePerIntegration** | **Integer**| the number of items per Integration to return per page. **Deprecated**. Use the 'size' query parameter instead. | [optional] [default to 20] | +| **page** | **Integer**| the requested page number | [optional] [default to 1] | +| **size** | **Integer**| the number of items to return per page | [optional] [default to 20] | +| **search** | **String**| returns only the items that match the search string | [optional] [default to null] | ### Return type -[**List**](../Models/Import.md) +[**findAllImports_200_response**](../Models/findAllImports_200_response.md) ### Authorization diff --git a/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md b/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md index 427a3a6b50..51e16e3672 100644 --- a/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md +++ b/plugins/bulk-import-backend/api-docs/Apis/OrganizationApi.md @@ -20,7 +20,7 @@ Fetch Organizations accessible by Backstage Github Integrations |------------- | ------------- | ------------- | -------------| | **pagePerIntegration** | **Integer**| the page number for each Integration | [optional] [default to 1] | | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | -| **search** | **String**| returns only organizations that match the search string, by name | [optional] [default to null] | +| **search** | **String**| returns only the items that match the search string | [optional] [default to null] | ### Return type @@ -49,7 +49,7 @@ Fetch Repositories in the specified GitHub organization, provided it is accessib | **checkImportStatus** | **Boolean**| whether to return import status. Note that this might incur a performance penalty because the import status is computed for each repository. | [optional] [default to false] | | **pagePerIntegration** | **Integer**| the page number for each Integration | [optional] [default to 1] | | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | -| **search** | **String**| returns only organization repositories that contain the search string, by repository name | [optional] [default to null] | +| **search** | **String**| returns only the items that match the search string | [optional] [default to null] | ### Return type diff --git a/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md b/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md index 9173eb3469..20ca62ff69 100644 --- a/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md +++ b/plugins/bulk-import-backend/api-docs/Apis/RepositoryApi.md @@ -20,7 +20,7 @@ Fetch Organization Repositories accessible by Backstage Github Integrations | **checkImportStatus** | **Boolean**| whether to return import status. Note that this might incur a performance penalty because the import status is computed for each repository. | [optional] [default to false] | | **pagePerIntegration** | **Integer**| the page number for each Integration | [optional] [default to 1] | | **sizePerIntegration** | **Integer**| the number of items per Integration to return per page | [optional] [default to 20] | -| **search** | **String**| returns only repositories that contain the search string, by name | [optional] [default to null] | +| **search** | **String**| returns only the items that match the search string | [optional] [default to null] | ### Return type diff --git a/plugins/bulk-import-backend/api-docs/Models/ImportJobListV2.md b/plugins/bulk-import-backend/api-docs/Models/ImportJobListV2.md new file mode 100644 index 0000000000..245c8a91ba --- /dev/null +++ b/plugins/bulk-import-backend/api-docs/Models/ImportJobListV2.md @@ -0,0 +1,13 @@ +# ImportJobListV2 +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **imports** | [**List**](Import.md) | | [optional] [default to null] | +| **errors** | **List** | | [optional] [default to null] | +| **totalCount** | **Integer** | | [optional] [default to null] | +| **page** | **Integer** | | [optional] [default to null] | +| **size** | **Integer** | | [optional] [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/plugins/bulk-import-backend/api-docs/Models/Import_repository.md b/plugins/bulk-import-backend/api-docs/Models/Import_repository.md deleted file mode 100644 index 3224ce2d51..0000000000 --- a/plugins/bulk-import-backend/api-docs/Models/Import_repository.md +++ /dev/null @@ -1,11 +0,0 @@ -# Import_repository -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -| **name** | **String** | repository name | [optional] [default to null] | -| **url** | **String** | repository URL | [optional] [default to null] | -| **organization** | **String** | organization which the repository is part of | [optional] [default to null] | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - diff --git a/plugins/bulk-import-backend/api-docs/Models/findAllImports_200_response.md b/plugins/bulk-import-backend/api-docs/Models/findAllImports_200_response.md new file mode 100644 index 0000000000..672048cc29 --- /dev/null +++ b/plugins/bulk-import-backend/api-docs/Models/findAllImports_200_response.md @@ -0,0 +1,13 @@ +# findAllImports_200_response +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **imports** | [**List**](Import.md) | | [optional] [default to null] | +| **errors** | **List** | | [optional] [default to null] | +| **totalCount** | **Integer** | | [optional] [default to null] | +| **page** | **Integer** | | [optional] [default to null] | +| **size** | **Integer** | | [optional] [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/plugins/bulk-import-backend/api-docs/Models/findAllImports_500_response.md b/plugins/bulk-import-backend/api-docs/Models/findAllImports_500_response.md new file mode 100644 index 0000000000..7e5a4f12f4 --- /dev/null +++ b/plugins/bulk-import-backend/api-docs/Models/findAllImports_500_response.md @@ -0,0 +1,13 @@ +# findAllImports_500_response +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **imports** | [**List**](Import.md) | | [optional] [default to null] | +| **errors** | **List** | | [optional] [default to null] | +| **totalCount** | **Integer** | | [optional] [default to null] | +| **page** | **Integer** | | [optional] [default to null] | +| **size** | **Integer** | | [optional] [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/plugins/bulk-import-backend/api-docs/README.md b/plugins/bulk-import-backend/api-docs/README.md index 0d3176381d..fc34e75f9a 100644 --- a/plugins/bulk-import-backend/api-docs/README.md +++ b/plugins/bulk-import-backend/api-docs/README.md @@ -22,6 +22,7 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* - [ApprovalTool](./Models/ApprovalTool.md) - [Import](./Models/Import.md) + - [ImportJobListV2](./Models/ImportJobListV2.md) - [ImportRequest](./Models/ImportRequest.md) - [ImportRequest_github](./Models/ImportRequest_github.md) - [ImportRequest_github_pullRequest](./Models/ImportRequest_github_pullRequest.md) @@ -33,6 +34,8 @@ All URIs are relative to *http://localhost:7007/api/bulk-import* - [OrganizationList](./Models/OrganizationList.md) - [Repository](./Models/Repository.md) - [RepositoryList](./Models/RepositoryList.md) + - [findAllImports_200_response](./Models/findAllImports_200_response.md) + - [findAllImports_500_response](./Models/findAllImports_500_response.md) - [ping_200_response](./Models/ping_200_response.md) diff --git a/plugins/bulk-import-backend/scripts/openapi.sh b/plugins/bulk-import-backend/scripts/openapi.sh index 05fc10eb69..ebce7ffa2a 100755 --- a/plugins/bulk-import-backend/scripts/openapi.sh +++ b/plugins/bulk-import-backend/scripts/openapi.sh @@ -41,7 +41,7 @@ echo '`' >> "${OPENAPI_DOC_JS_FILE}" echo "export const openApiDocument = JSON.parse(OPENAPI);" >> "${OPENAPI_DOC_JS_FILE}" rm -f ./src/schema/openapi.json -# Generate doc -#npx --yes --package=openapicmd@2.3.2 -- openapi redoc src/schema/openapi.yaml --bundle docs +# Re-generate doc +rm -rf ./api-docs/ npx --yes --package=@openapitools/openapi-generator-cli@2.13.4 -- \ openapi-generator-cli generate -i ./src/schema/openapi.yaml -g markdown -o ./api-docs/ diff --git a/plugins/bulk-import-backend/src/helpers/catalogInfoGenerator.ts b/plugins/bulk-import-backend/src/helpers/catalogInfoGenerator.ts index 6f4bbe3e7a..77b556c24e 100644 --- a/plugins/bulk-import-backend/src/helpers/catalogInfoGenerator.ts +++ b/plugins/bulk-import-backend/src/helpers/catalogInfoGenerator.ts @@ -125,18 +125,21 @@ ${jsYaml.dump(generatedEntity.entity)}`, search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, - ): Promise { - const list = await this.listCatalogUrlLocationsById( + ): Promise<{ targetUrls: string[]; totalCount?: number }> { + const byId = await this.listCatalogUrlLocationsById( config, search, pageNumber, pageSize, ); const result = new Set(); - for (const l of list) { + for (const l of byId.locations) { result.add(l.target); } - return Array.from(result.values()); + return { + targetUrls: Array.from(result.values()), + totalCount: byId.totalCount, + }; } async listCatalogUrlLocationsById( @@ -144,18 +147,32 @@ ${jsYaml.dump(generatedEntity.entity)}`, search?: string, pageNumber: number = DefaultPageNumber, pageSize: number = DefaultPageSize, - ): Promise<{ id?: string; target: string }[]> { + ): Promise<{ + locations: { id?: string; target: string }[]; + totalCount?: number; + }> { const result = await Promise.all([ this.listCatalogUrlLocationsFromConfig(config, search), this.listCatalogUrlLocationsByIdFromLocationsEndpoint(search), this.listCatalogUrlLocationEntitiesById(search, pageNumber, pageSize), ]); - return result.flat(); + const locations = result.flatMap(u => u.locations); + // we might have duplicate elements here + const totalCount = result + .map(l => l.totalCount ?? 0) + .reduce((accumulator, currentValue) => accumulator + currentValue, 0); + return { + locations, + totalCount, + }; } async listCatalogUrlLocationsByIdFromLocationsEndpoint( search?: string, - ): Promise<{ id?: string; target: string }[]> { + ): Promise<{ + locations: { id?: string; target: string }[]; + totalCount?: number; + }> { const url = `${await this.discovery.getBaseUrl('catalog')}/locations`; const response = await fetch(url, { headers: { @@ -168,7 +185,7 @@ ${jsYaml.dump(generatedEntity.entity)}`, data: { id: string; target: string; type: string }; }[]; if (!Array.isArray(locations)) { - return []; + return { locations: [] }; } const res = locations .filter( @@ -180,13 +197,14 @@ ${jsYaml.dump(generatedEntity.entity)}`, target: location.data.target, }; }); - return this.filterLocations(res, search); + const filtered = this.filterLocations(res, search); + return { locations: filtered, totalCount: filtered.length }; } listCatalogUrlLocationsFromConfig( config: Config, search?: string, - ): { id?: string; target: string }[] { + ): { locations: { id?: string; target: string }[]; totalCount?: number } { const locationConfigs = config.getOptionalConfigArray('catalog.locations') ?? []; const res = locationConfigs @@ -202,14 +220,18 @@ ${jsYaml.dump(generatedEntity.entity)}`, target, }; }); - return this.filterLocations(res, search); + const filtered = this.filterLocations(res, search); + return { locations: filtered, totalCount: filtered.length }; } async listCatalogUrlLocationEntitiesById( search?: string, _pageNumber: number = DefaultPageNumber, _pageSize: number = DefaultPageSize, - ): Promise<{ id?: string; target: string }[]> { + ): Promise<{ + locations: { id?: string; target: string }[]; + totalCount?: number; + }> { const result = await this.catalogApi.getEntities( { filter: { @@ -217,8 +239,9 @@ ${jsYaml.dump(generatedEntity.entity)}`, }, // There is no query parameter to find entities with target URLs containing a string. // The existing filter does an exact matching. That's why we are retrieving this hard-coded high number of Locations. - limit: 1000, + limit: 9999, offset: 0, + order: { field: 'metadata.name', order: 'desc' }, }, { token: await getTokenForPlugin(this.auth, 'catalog'), @@ -235,7 +258,8 @@ ${jsYaml.dump(generatedEntity.entity)}`, target: location.spec.target!, }; }); - return this.filterLocations(res, search); + const filtered = this.filterLocations(res, search); + return { locations: filtered, totalCount: filtered.length }; } private filterLocations( diff --git a/plugins/bulk-import-backend/src/openapi.d.ts b/plugins/bulk-import-backend/src/openapi.d.ts index 2882acce88..2533b535cf 100644 --- a/plugins/bulk-import-backend/src/openapi.d.ts +++ b/plugins/bulk-import-backend/src/openapi.d.ts @@ -11,6 +11,28 @@ import type { } from 'openapi-client-axios'; declare namespace Components { + export interface HeaderParameters { + apiVersionHeaderParam?: Parameters.ApiVersionHeaderParam; + } + namespace Parameters { + export type ApiVersionHeaderParam = "v1" | "v2"; + export type PagePerIntegrationQueryParam = number; + export type PagePerIntegrationQueryParamDeprecated = number; + export type PageQueryParam = number; + export type SearchQueryParam = string; + export type SizePerIntegrationQueryParam = number; + export type SizePerIntegrationQueryParamDeprecated = number; + export type SizeQueryParam = number; + } + export interface QueryParameters { + pagePerIntegrationQueryParam?: Parameters.PagePerIntegrationQueryParam; + sizePerIntegrationQueryParam?: Parameters.SizePerIntegrationQueryParam; + pagePerIntegrationQueryParamDeprecated?: Parameters.PagePerIntegrationQueryParamDeprecated; + sizePerIntegrationQueryParamDeprecated?: Parameters.SizePerIntegrationQueryParamDeprecated; + searchQueryParam?: Parameters.SearchQueryParam; + pageQueryParam?: Parameters.PageQueryParam; + sizeQueryParam?: Parameters.SizeQueryParam; + } namespace Schemas { export type ApprovalTool = "GIT" | "SERVICENOW"; /** @@ -55,6 +77,16 @@ declare namespace Components { }; }; } + /** + * Import Job List + */ + export interface ImportJobListV2 { + imports?: /* Import Job */ Import[]; + errors?: string[]; + totalCount?: number; + page?: number; + size?: number; + } /** * Import Job request */ @@ -219,20 +251,27 @@ declare namespace Paths { } } namespace FindAllImports { + export interface HeaderParameters { + "api-version"?: Parameters.ApiVersion; + } namespace Parameters { + export type ApiVersion = "v1" | "v2"; + export type Page = number; export type PagePerIntegration = number; export type Search = string; + export type Size = number; export type SizePerIntegration = number; } export interface QueryParameters { pagePerIntegration?: Parameters.PagePerIntegration; sizePerIntegration?: Parameters.SizePerIntegration; + page?: Parameters.Page; + size?: Parameters.Size; search?: Parameters.Search; } namespace Responses { - export type $200 = /* Import Job */ Components.Schemas.Import[]; - export interface $500 { - } + export type $200 = /* Import Job */ Components.Schemas.Import[] | /* Import Job List */ Components.Schemas.ImportJobListV2; + export type $500 = string | /* Import Job List */ Components.Schemas.ImportJobListV2; } } namespace FindAllOrganizations { @@ -352,7 +391,7 @@ export interface OperationMethods { * findAllImports - Fetch Import Jobs */ 'findAllImports'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -428,7 +467,7 @@ export interface PathsDictionary { * findAllImports - Fetch Import Jobs */ 'get'( - parameters?: Parameters | null, + parameters?: Parameters | null, data?: any, config?: AxiosRequestConfig ): OperationResponse @@ -463,11 +502,3 @@ export interface PathsDictionary { export type Client = OpenAPIClient -export type ApprovalTool = Components.Schemas.ApprovalTool; -export type Import = Components.Schemas.Import; -export type ImportRequest = Components.Schemas.ImportRequest; -export type ImportStatus = Components.Schemas.ImportStatus; -export type Organization = Components.Schemas.Organization; -export type OrganizationList = Components.Schemas.OrganizationList; -export type Repository = Components.Schemas.Repository; -export type RepositoryList = Components.Schemas.RepositoryList; diff --git a/plugins/bulk-import-backend/src/openapidocument.ts b/plugins/bulk-import-backend/src/openapidocument.ts index 3c2228811c..f416447471 100644 --- a/plugins/bulk-import-backend/src/openapidocument.ts +++ b/plugins/bulk-import-backend/src/openapidocument.ts @@ -79,30 +79,13 @@ const OPENAPI = ` ], "parameters": [ { - "in": "query", - "name": "pagePerIntegration", - "description": "the page number for each Integration", - "schema": { - "type": "integer", - "default": 1 - } + "$ref": "#/components/parameters/pagePerIntegrationQueryParam" }, { - "in": "query", - "name": "sizePerIntegration", - "description": "the number of items per Integration to return per page", - "schema": { - "type": "integer", - "default": 20 - } + "$ref": "#/components/parameters/sizePerIntegrationQueryParam" }, { - "in": "query", - "name": "search", - "description": "returns only organizations that match the search string, by name", - "schema": { - "type": "string" - } + "$ref": "#/components/parameters/searchQueryParam" } ], "responses": { @@ -171,30 +154,13 @@ const OPENAPI = ` } }, { - "in": "query", - "name": "pagePerIntegration", - "description": "the page number for each Integration", - "schema": { - "type": "integer", - "default": 1 - } + "$ref": "#/components/parameters/pagePerIntegrationQueryParam" }, { - "in": "query", - "name": "sizePerIntegration", - "description": "the number of items per Integration to return per page", - "schema": { - "type": "integer", - "default": 20 - } + "$ref": "#/components/parameters/sizePerIntegrationQueryParam" }, { - "in": "query", - "name": "search", - "description": "returns only organization repositories that contain the search string, by repository name", - "schema": { - "type": "string" - } + "$ref": "#/components/parameters/searchQueryParam" } ], "responses": { @@ -254,30 +220,13 @@ const OPENAPI = ` } }, { - "in": "query", - "name": "pagePerIntegration", - "description": "the page number for each Integration", - "schema": { - "type": "integer", - "default": 1 - } + "$ref": "#/components/parameters/pagePerIntegrationQueryParam" }, { - "in": "query", - "name": "sizePerIntegration", - "description": "the number of items per Integration to return per page", - "schema": { - "type": "integer", - "default": 20 - } + "$ref": "#/components/parameters/sizePerIntegrationQueryParam" }, { - "in": "query", - "name": "search", - "description": "returns only repositories that contain the search string, by name", - "schema": { - "type": "string" - } + "$ref": "#/components/parameters/searchQueryParam" } ], "responses": { @@ -328,53 +277,75 @@ const OPENAPI = ` ], "parameters": [ { - "in": "query", - "name": "pagePerIntegration", - "description": "the page number for each Integration", - "schema": { - "type": "integer", - "default": 1 - } + "$ref": "#/components/parameters/apiVersionHeaderParam" }, { - "in": "query", - "name": "sizePerIntegration", - "description": "the number of items per Integration to return per page", - "schema": { - "type": "integer", - "default": 20 - } + "$ref": "#/components/parameters/pagePerIntegrationQueryParamDeprecated" }, { - "in": "query", - "name": "search", - "description": "returns only Imports that contain the search string, by repository name", - "schema": { - "type": "string" - } + "$ref": "#/components/parameters/sizePerIntegrationQueryParamDeprecated" + }, + { + "$ref": "#/components/parameters/pageQueryParam" + }, + { + "$ref": "#/components/parameters/sizeQueryParam" + }, + { + "$ref": "#/components/parameters/searchQueryParam" } ], "responses": { "200": { - "description": "Import Jobs list was fetched successfully with no errors", + "description": "Import Job list was fetched successfully with no errors", "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Import" - } + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/components/schemas/Import" + } + }, + { + "$ref": "#/components/schemas/ImportJobListV2" + } + ] }, "examples": { "twoImports": { "$ref": "#/components/examples/twoImports" + }, + "multipleImportJobsV2": { + "$ref": "#/components/examples/multipleImportJobsV2" } } } } }, "500": { - "description": "Generic error" + "description": "Generic error when there are errors and no Import Job is returned", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "type": "string", + "description": "Generic error" + }, + { + "$ref": "#/components/schemas/ImportJobListV2" + } + ] + }, + "examples": { + "repositoryListErrors": { + "$ref": "#/components/examples/importJobListErrors" + } + } + } + } } } }, @@ -432,7 +403,7 @@ const OPENAPI = ` }, "examples": { "twoImports": { - "$ref": "#/components/examples/twoImports" + "$ref": "#/components/examples/twoImportJobs" } } } @@ -535,6 +506,85 @@ const OPENAPI = ` } }, "components": { + "parameters": { + "apiVersionHeaderParam": { + "in": "header", + "name": "api-version", + "description": "API version.\\n\\n## Changelog\\n\\n### v1 (default)\\nInitial version\\n#### Deprecations\\n* GET /imports\\n * Deprecation of 'pagePerIntegration' and 'sizePerIntegration' query parameters and introduction of new 'page' and 'size' parameters\\n * 'page' takes precedence over 'pagePerIntegration' if both are passed\\n * 'size' takes precedence over 'sizePerIntegration' if both are passed\\n\\n### v2\\n#### Breaking changes\\n* GET /imports\\n * Query parameters:\\n * 'pagePerIntegration' is ignored in favor of 'page'\\n * 'sizePerIntegration' is ignored in favor of 'size'\\n * Response structure changed to include pagination info: instead of returning a simple list of Imports, the response is now an object containing the following fields:\\n * 'imports': the list of Imports\\n * 'page': the page requested\\n * 'size': the requested number of Imports requested per page\\n * 'totalCount': the total count of Imports\\n", + "schema": { + "type": "string", + "enum": [ + "v1", + "v2" + ], + "default": "v1" + } + }, + "pagePerIntegrationQueryParam": { + "in": "query", + "name": "pagePerIntegration", + "description": "the page number for each Integration", + "schema": { + "type": "integer", + "default": 1 + } + }, + "sizePerIntegrationQueryParam": { + "in": "query", + "name": "sizePerIntegration", + "description": "the number of items per Integration to return per page", + "schema": { + "type": "integer", + "default": 20 + } + }, + "pagePerIntegrationQueryParamDeprecated": { + "in": "query", + "name": "pagePerIntegration", + "description": "the page number for each Integration. **Deprecated**. Use the 'page' query parameter instead.", + "deprecated": true, + "schema": { + "type": "integer", + "default": 1 + } + }, + "sizePerIntegrationQueryParamDeprecated": { + "in": "query", + "name": "sizePerIntegration", + "description": "the number of items per Integration to return per page. **Deprecated**. Use the 'size' query parameter instead.", + "deprecated": true, + "schema": { + "type": "integer", + "default": 20 + } + }, + "searchQueryParam": { + "in": "query", + "name": "search", + "description": "returns only the items that match the search string", + "schema": { + "type": "string" + } + }, + "pageQueryParam": { + "in": "query", + "name": "page", + "description": "the requested page number", + "schema": { + "type": "integer", + "default": 1 + } + }, + "sizeQueryParam": { + "in": "query", + "name": "size", + "description": "the number of items to return per page", + "schema": { + "type": "integer", + "default": 20 + } + } + }, "schemas": { "OrganizationList": { "title": "Organization List", @@ -679,6 +729,33 @@ const OPENAPI = ` null ] }, + "ImportJobListV2": { + "title": "Import Job List", + "type": "object", + "properties": { + "imports": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Import" + } + }, + "errors": { + "type": "array", + "items": { + "type": "string" + } + }, + "totalCount": { + "type": "integer" + }, + "page": { + "type": "integer" + }, + "size": { + "type": "integer" + } + } + }, "Import": { "title": "Import Job", "type": "object", @@ -898,7 +975,100 @@ const OPENAPI = ` } }, "twoImports": { - "summary": "Two import job requests", + "summary": "Two import job requests (V1)", + "value": [ + { + "id": "bulk-import-id-1", + "status": "WAIT_PR_APPROVAL", + "errors": [], + "approvalTool": "GIT", + "repository": { + "name": "pet-app", + "url": "https://github.com/my-org/pet-app", + "organization": "my-org" + }, + "github": { + "pullRequest": { + "url": "https://github.com/my-org/pet-app/pull/1", + "number": 1 + } + } + }, + { + "id": "bulk-import-id-2", + "status": "PR_REJECTED", + "errors": [], + "approvalTool": "GIT", + "repository": { + "name": "pet-app-test", + "url": "https://github.com/my-org/pet-app-test", + "organization": "my-org" + }, + "github": { + "pullRequest": { + "url": "https://github.com/my-org/pet-app-test/pull/10", + "number": 10 + } + } + } + ] + }, + "multipleImportJobsV2": { + "summary": "Two import job requests (V2)", + "value": { + "errors": [], + "page": 1, + "size": 2, + "totalCount": 10, + "imports": [ + { + "id": "bulk-import-id-1", + "status": "WAIT_PR_APPROVAL", + "errors": [], + "approvalTool": "GIT", + "repository": { + "name": "pet-app", + "url": "https://github.com/my-org/pet-app", + "organization": "my-org" + }, + "github": { + "pullRequest": { + "url": "https://github.com/my-org/pet-app/pull/1", + "number": 1 + } + } + }, + { + "id": "bulk-import-id-2", + "status": "PR_REJECTED", + "errors": [], + "approvalTool": "GIT", + "repository": { + "name": "pet-app-test", + "url": "https://github.com/my-org/pet-app-test", + "organization": "my-org" + }, + "github": { + "pullRequest": { + "url": "https://github.com/my-org/pet-app-test/pull/10", + "number": 10 + } + } + } + ] + } + }, + "importJobListErrors": { + "summary": "Errors when listing import jobs", + "value": { + "errors": [ + "Github App with ID xyz-123 failed spectacularly" + ], + "imports": [] + } + }, + "twoImportJobs": { + "summary": "Two import jobs", "value": [ { "id": "bulk-import-id-1", diff --git a/plugins/bulk-import-backend/src/schema/openapi.yaml b/plugins/bulk-import-backend/src/schema/openapi.yaml index 994da75955..fb0e3ee712 100644 --- a/plugins/bulk-import-backend/src/schema/openapi.yaml +++ b/plugins/bulk-import-backend/src/schema/openapi.yaml @@ -57,23 +57,9 @@ paths: - BearerAuth: [] tags: [Organization] parameters: - - in: query - name: pagePerIntegration - description: the page number for each Integration - schema: - type: integer - default: 1 - - in: query - name: sizePerIntegration - description: the number of items per Integration to return per page - schema: - type: integer - default: 20 - - in: query - name: search - description: returns only organizations that match the search string, by name - schema: - type: string + - $ref: '#/components/parameters/pagePerIntegrationQueryParam' + - $ref: '#/components/parameters/sizePerIntegrationQueryParam' + - $ref: '#/components/parameters/searchQueryParam' responses: 200: description: Organization list was fetched successfully with no errors @@ -114,23 +100,9 @@ paths: schema: type: boolean default: 'false' - - in: query - name: pagePerIntegration - description: the page number for each Integration - schema: - type: integer - default: 1 - - in: query - name: sizePerIntegration - description: the number of items per Integration to return per page - schema: - type: integer - default: 20 - - in: query - name: search - description: returns only organization repositories that contain the search string, by repository name - schema: - type: string + - $ref: '#/components/parameters/pagePerIntegrationQueryParam' + - $ref: '#/components/parameters/sizePerIntegrationQueryParam' + - $ref: '#/components/parameters/searchQueryParam' responses: 200: description: Org Repository list was fetched successfully with no errors @@ -165,23 +137,9 @@ paths: schema: type: boolean default: 'false' - - in: query - name: pagePerIntegration - description: the page number for each Integration - schema: - type: integer - default: 1 - - in: query - name: sizePerIntegration - description: the number of items per Integration to return per page - schema: - type: integer - default: 20 - - in: query - name: search - description: returns only repositories that contain the search string, by name - schema: - type: string + - $ref: '#/components/parameters/pagePerIntegrationQueryParam' + - $ref: '#/components/parameters/sizePerIntegrationQueryParam' + - $ref: '#/components/parameters/searchQueryParam' responses: 200: description: Repository list was fetched successfully with no errors @@ -210,37 +168,45 @@ paths: - BearerAuth: [] tags: [Import] parameters: - - in: query - name: pagePerIntegration - description: the page number for each Integration - schema: - type: integer - default: 1 - - in: query - name: sizePerIntegration - description: the number of items per Integration to return per page - schema: - type: integer - default: 20 - - in: query - name: search - description: returns only Imports that contain the search string, by repository name - schema: - type: string + - $ref: '#/components/parameters/apiVersionHeaderParam' + + # The '*PerIntegration' query params are being kept for backward compatibility, + # but the behavior depends on the API Version specified in the request headers. + - $ref: '#/components/parameters/pagePerIntegrationQueryParamDeprecated' + - $ref: '#/components/parameters/sizePerIntegrationQueryParamDeprecated' + - $ref: '#/components/parameters/pageQueryParam' + - $ref: '#/components/parameters/sizeQueryParam' + + - $ref: '#/components/parameters/searchQueryParam' + responses: 200: - description: Import Jobs list was fetched successfully with no errors + description: Import Job list was fetched successfully with no errors content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/Import' + oneOf: + - type: array + items: + $ref: '#/components/schemas/Import' + - $ref: '#/components/schemas/ImportJobListV2' examples: twoImports: $ref: '#/components/examples/twoImports' + multipleImportJobsV2: + $ref: '#/components/examples/multipleImportJobsV2' 500: - description: Generic error + description: Generic error when there are errors and no Import Job is returned + content: + application/json: + schema: + oneOf: + - type: string + description: Generic error + - $ref: '#/components/schemas/ImportJobListV2' + examples: + repositoryListErrors: + $ref: '#/components/examples/importJobListErrors' post: operationId: createImportJobs @@ -280,7 +246,7 @@ paths: $ref: '#/components/schemas/Import' examples: twoImports: - $ref: '#/components/examples/twoImports' + $ref: '#/components/examples/twoImportJobs' /import/by-repo: get: @@ -339,6 +305,92 @@ paths: description: Generic error components: + parameters: + apiVersionHeaderParam: + in: header + name: api-version + description: | + API version. + + ## Changelog + + ### v1 (default) + Initial version + #### Deprecations + * GET /imports + * Deprecation of 'pagePerIntegration' and 'sizePerIntegration' query parameters and introduction of new 'page' and 'size' parameters + * 'page' takes precedence over 'pagePerIntegration' if both are passed + * 'size' takes precedence over 'sizePerIntegration' if both are passed + + ### v2 + #### Breaking changes + * GET /imports + * Query parameters: + * 'pagePerIntegration' is ignored in favor of 'page' + * 'sizePerIntegration' is ignored in favor of 'size' + * Response structure changed to include pagination info: instead of returning a simple list of Imports, the response is now an object containing the following fields: + * 'imports': the list of Imports + * 'page': the page requested + * 'size': the requested number of Imports requested per page + * 'totalCount': the total count of Imports + schema: + type: string + enum: ['v1', 'v2'] + default: 'v1' + + pagePerIntegrationQueryParam: + in: query + name: pagePerIntegration + description: the page number for each Integration + schema: + type: integer + default: 1 + sizePerIntegrationQueryParam: + in: query + name: sizePerIntegration + description: the number of items per Integration to return per page + schema: + type: integer + default: 20 + pagePerIntegrationQueryParamDeprecated: + in: query + name: pagePerIntegration + description: the page number for each Integration. **Deprecated**. Use the 'page' query parameter instead. + deprecated: true + schema: + type: integer + default: 1 + sizePerIntegrationQueryParamDeprecated: + in: query + name: sizePerIntegration + description: the number of items per Integration to return per page. **Deprecated**. Use the 'size' query parameter instead. + deprecated: true + schema: + type: integer + default: 20 + searchQueryParam: + in: query + name: search + description: returns only the items that match the search string + schema: + type: string + pageQueryParam: + # used for endpoints where pagination does not depend on the Integrations configured + in: query + name: page + description: the requested page number + schema: + type: integer + default: 1 + sizeQueryParam: + # used for endpoints where pagination does not depend on the Integrations configured + in: query + name: size + description: the number of items to return per page + schema: + type: integer + default: 20 + schemas: OrganizationList: title: Organization List @@ -454,6 +506,25 @@ components: #- SERVICENOW_TICKET_REJECTED - null + ImportJobListV2: + title: Import Job List + type: object + properties: + imports: + type: array + items: + $ref: '#/components/schemas/Import' + errors: + type: array + items: + type: string + totalCount: + type: integer + page: + type: integer + size: + type: integer + Import: title: Import Job type: object @@ -614,7 +685,75 @@ components: repositories: [] twoImports: - summary: Two import job requests + summary: Two import job requests (V1) + value: + - id: 'bulk-import-id-1' + status: 'WAIT_PR_APPROVAL' + errors: [] + approvalTool: GIT + repository: + name: 'pet-app' + url: 'https://github.com/my-org/pet-app' + organization: 'my-org' + github: + pullRequest: + url: 'https://github.com/my-org/pet-app/pull/1' + number: 1 + - id: 'bulk-import-id-2' + status: 'PR_REJECTED' + errors: [] + approvalTool: GIT + repository: + name: 'pet-app-test' + url: 'https://github.com/my-org/pet-app-test' + organization: 'my-org' + github: + pullRequest: + url: 'https://github.com/my-org/pet-app-test/pull/10' + number: 10 + + multipleImportJobsV2: + summary: Two import job requests (V2) + value: + errors: [] + page: 1 + size: 2 + totalCount: 10 + imports: + - id: 'bulk-import-id-1' + status: 'WAIT_PR_APPROVAL' + errors: [] + approvalTool: GIT + repository: + name: 'pet-app' + url: 'https://github.com/my-org/pet-app' + organization: 'my-org' + github: + pullRequest: + url: 'https://github.com/my-org/pet-app/pull/1' + number: 1 + - id: 'bulk-import-id-2' + status: 'PR_REJECTED' + errors: [] + approvalTool: GIT + repository: + name: 'pet-app-test' + url: 'https://github.com/my-org/pet-app-test' + organization: 'my-org' + github: + pullRequest: + url: 'https://github.com/my-org/pet-app-test/pull/10' + number: 10 + + importJobListErrors: + summary: Errors when listing import jobs + value: + errors: + - 'Github App with ID xyz-123 failed spectacularly' + imports: [] + + twoImportJobs: + summary: Two import jobs value: - id: 'bulk-import-id-1' status: 'WAIT_PR_APPROVAL' diff --git a/plugins/bulk-import-backend/src/service/handlers/bulkImports.test.ts b/plugins/bulk-import-backend/src/service/handlers/bulkImports.test.ts index 7c2c43b173..d9436e96ee 100644 --- a/plugins/bulk-import-backend/src/service/handlers/bulkImports.test.ts +++ b/plugins/bulk-import-backend/src/service/handlers/bulkImports.test.ts @@ -17,10 +17,12 @@ import type { LoggerService } from '@backstage/backend-plugin-api'; import { mockServices } from '@backstage/backend-test-utils'; import type { CatalogClient } from '@backstage/catalog-client'; +import type { Config } from '@backstage/config'; import gitUrlParse from 'git-url-parse'; import { CatalogInfoGenerator } from '../../helpers'; +import { Paths } from '../../openapi'; import { GithubApiService } from '../githubApiService'; import { deleteImportByRepo, findAllImports } from './bulkImports'; @@ -42,7 +44,7 @@ const config = mockServices.rootConfig({ clientSecret: 'CLIENT_SECRET', }, ], - token: 'hardcoded_token', + token: 'hardcoded_token', // notsecret }, ], }, @@ -159,143 +161,446 @@ describe('bulkimports.ts tests', () => { } describe('findAllImports', () => { - it('should return only imports from repos that are accessible from the configured GH integrations', async () => { - jest - .spyOn(mockCatalogInfoGenerator, 'listCatalogUrlLocations') - .mockResolvedValue([ - // from app-config - 'https://github.com/my-org-1/my-repo-11/blob/main/catalog-info.yaml', - 'https://github.com/my-org-1/my-repo-12/blob/main/some/path/to/catalog-info.yaml', - 'https://github.com/my-user/my-repo-123/blob/main/catalog-info.yaml', - 'https://github.com/some-public-org/some-public-repo/blob/main/catalog-info.yaml', + const locationUrls = [ + // from app-config + 'https://github.com/my-org-1/my-repo-11/blob/main/catalog-info.yaml', + 'https://github.com/my-org-1/my-repo-12/blob/main/some/path/to/catalog-info.yaml', + 'https://github.com/my-user/my-repo-123/blob/main/catalog-info.yaml', + 'https://github.com/some-public-org/some-public-repo/blob/main/catalog-info.yaml', - // from some Locations - 'https://github.com/my-org-2/my-repo-21/blob/master/catalog-info.yaml', - 'https://github.com/my-org-2/my-repo-22/blob/master/catalog-info.yaml', - 'https://github.com/my-org-21/my-repo-211/blob/another-branch/catalog-info.yaml', + // from some Locations + 'https://github.com/my-org-2/my-repo-21/blob/master/catalog-info.yaml', + 'https://github.com/my-org-2/my-repo-22/blob/master/catalog-info.yaml', + 'https://github.com/my-org-21/my-repo-211/blob/another-branch/catalog-info.yaml', - // from some Location entities (simulating repos that could be auto-discovered by the discovery plugin) - 'https://github.com/my-org-3/my-repo-31/blob/main/catalog-info.yaml', - 'https://github.com/my-org-3/my-repo-32/blob/dev/catalog-info.yaml', - 'https://github.com/my-org-3/my-repo-33/blob/dev/all.yaml', - 'https://github.com/my-org-3/my-repo-34/blob/dev/path/to/catalog-info.yaml', - ]); - jest - .spyOn( - mockGithubApiService, - 'filterLocationsAccessibleFromIntegrations', - ) - .mockResolvedValue([ - // only repos that are accessible from the configured GH integrations - // are considered as valid Imports - 'https://github.com/my-org-1/my-repo-11/blob/main/catalog-info.yaml', // PR - 'https://github.com/my-user/my-repo-123/blob/main/catalog-info.yaml', // PR Error - 'https://github.com/my-org-2/my-repo-21/blob/master/catalog-info.yaml', // ADDED - 'https://github.com/my-org-2/my-repo-22/blob/master/catalog-info.yaml', // no PR => null status - 'https://github.com/my-org-3/my-repo-31/blob/main/catalog-info.yaml', // ADDED - 'https://github.com/my-org-3/my-repo-32/blob/dev/catalog-info.yaml', // PR - ]); - jest - .spyOn(mockCatalogInfoGenerator, 'findLocationEntitiesByTargetUrl') - .mockResolvedValue([]); + // from some Location entities (simulating repos that could be auto-discovered by the discovery plugin) + 'https://github.com/my-org-3/my-repo-31/blob/main/catalog-info.yaml', + 'https://github.com/my-org-3/my-repo-32/blob/dev/catalog-info.yaml', + 'https://github.com/my-org-3/my-repo-33/blob/dev/all.yaml', + 'https://github.com/my-org-3/my-repo-34/blob/dev/path/to/catalog-info.yaml', + ]; - const resp = await findAllImports( - logger, - config, - mockGithubApiService, - mockCatalogInfoGenerator, - ); - expect(resp.statusCode).toEqual(200); - expect(resp.responseBody).toEqual([ - { - id: 'https://github.com/my-org-1/my-repo-11', - repository: { - url: 'https://github.com/my-org-1/my-repo-11', - name: 'my-repo-11', - organization: 'my-org-1', - id: 'my-org-1/my-repo-11', - defaultBranch: 'main', + it.each([undefined, 'v1', 'v2'])( + 'should return only imports from repos that are accessible from the configured GH integrations (API Version: %s)', + async apiVersionStr => { + jest + .spyOn(mockCatalogInfoGenerator, 'listCatalogUrlLocations') + .mockResolvedValue({ + targetUrls: locationUrls, + totalCount: locationUrls.length, + }); + jest + .spyOn( + mockGithubApiService, + 'filterLocationsAccessibleFromIntegrations', + ) + .mockResolvedValue([ + // only repos that are accessible from the configured GH integrations + // are considered as valid Imports + 'https://github.com/my-org-1/my-repo-11/blob/main/catalog-info.yaml', // PR + 'https://github.com/my-user/my-repo-123/blob/main/catalog-info.yaml', // PR Error + 'https://github.com/my-org-2/my-repo-21/blob/master/catalog-info.yaml', // ADDED + 'https://github.com/my-org-2/my-repo-22/blob/master/catalog-info.yaml', // no PR => null status + 'https://github.com/my-org-3/my-repo-31/blob/main/catalog-info.yaml', // ADDED + 'https://github.com/my-org-3/my-repo-32/blob/dev/catalog-info.yaml', // PR + ]); + jest + .spyOn(mockCatalogInfoGenerator, 'findLocationEntitiesByTargetUrl') + .mockResolvedValue([]); + + const apiVersion = apiVersionStr as + | Paths.FindAllImports.Parameters.ApiVersion + | undefined; + let resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, }, - approvalTool: 'GIT', - status: 'WAIT_PR_APPROVAL', - github: { - pullRequest: { - number: 987, - url: 'https://github.com/my-org-1/my-repo-11/pull/987', + ); + expect(resp.statusCode).toEqual(200); + const allImportsExpected = [ + { + id: 'https://github.com/my-org-1/my-repo-11', + repository: { + url: 'https://github.com/my-org-1/my-repo-11', + name: 'my-repo-11', + organization: 'my-org-1', + id: 'my-org-1/my-repo-11', + defaultBranch: 'main', + }, + approvalTool: 'GIT', + status: 'WAIT_PR_APPROVAL', + github: { + pullRequest: { + number: 987, + url: 'https://github.com/my-org-1/my-repo-11/pull/987', + }, }, }, - }, - { - id: 'https://github.com/my-user/my-repo-123', - repository: { - url: 'https://github.com/my-user/my-repo-123', - name: 'my-repo-123', - organization: 'my-user', - id: 'my-user/my-repo-123', - defaultBranch: 'main', + { + id: 'https://github.com/my-user/my-repo-123', + repository: { + url: 'https://github.com/my-user/my-repo-123', + name: 'my-repo-123', + organization: 'my-user', + id: 'my-user/my-repo-123', + defaultBranch: 'main', + }, + approvalTool: 'GIT', + status: 'PR_ERROR', + errors: [ + 'could not find out if there is an import PR open on this repo', + ], }, - approvalTool: 'GIT', - status: 'PR_ERROR', - errors: [ - 'could not find out if there is an import PR open on this repo', - ], - }, - { - id: 'https://github.com/my-org-2/my-repo-21', - repository: { - url: 'https://github.com/my-org-2/my-repo-21', - name: 'my-repo-21', - organization: 'my-org-2', - id: 'my-org-2/my-repo-21', - defaultBranch: 'master', + { + id: 'https://github.com/my-org-2/my-repo-21', + repository: { + url: 'https://github.com/my-org-2/my-repo-21', + name: 'my-repo-21', + organization: 'my-org-2', + id: 'my-org-2/my-repo-21', + defaultBranch: 'master', + }, + approvalTool: 'GIT', + status: 'ADDED', }, - approvalTool: 'GIT', - status: 'ADDED', - }, - { - id: 'https://github.com/my-org-2/my-repo-22', - repository: { - url: 'https://github.com/my-org-2/my-repo-22', - name: 'my-repo-22', - organization: 'my-org-2', - id: 'my-org-2/my-repo-22', - defaultBranch: 'master', + { + id: 'https://github.com/my-org-2/my-repo-22', + repository: { + url: 'https://github.com/my-org-2/my-repo-22', + name: 'my-repo-22', + organization: 'my-org-2', + id: 'my-org-2/my-repo-22', + defaultBranch: 'master', + }, + approvalTool: 'GIT', + status: null, }, - approvalTool: 'GIT', - status: null, - }, - { - id: 'https://github.com/my-org-3/my-repo-31', - repository: { - url: 'https://github.com/my-org-3/my-repo-31', - name: 'my-repo-31', - organization: 'my-org-3', - id: 'my-org-3/my-repo-31', - defaultBranch: 'main', + { + id: 'https://github.com/my-org-3/my-repo-31', + repository: { + url: 'https://github.com/my-org-3/my-repo-31', + name: 'my-repo-31', + organization: 'my-org-3', + id: 'my-org-3/my-repo-31', + defaultBranch: 'main', + }, + approvalTool: 'GIT', + status: 'ADDED', }, - approvalTool: 'GIT', - status: 'ADDED', - }, - { - id: 'https://github.com/my-org-3/my-repo-32', - repository: { - url: 'https://github.com/my-org-3/my-repo-32', - name: 'my-repo-32', - organization: 'my-org-3', - id: 'my-org-3/my-repo-32', - defaultBranch: 'dev', + { + id: 'https://github.com/my-org-3/my-repo-32', + repository: { + url: 'https://github.com/my-org-3/my-repo-32', + name: 'my-repo-32', + organization: 'my-org-3', + id: 'my-org-3/my-repo-32', + defaultBranch: 'dev', + }, + approvalTool: 'GIT', + status: 'WAIT_PR_APPROVAL', + github: { + pullRequest: { + number: 100, + url: 'https://github.com/my-org-2/my-repo-21/pull/100', + }, + }, + }, + ]; + let expectedResponse: any = allImportsExpected; + if (apiVersion === 'v2') { + expectedResponse = { + imports: allImportsExpected, + page: 1, + size: 20, + totalCount: 6, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + + // Request different pages and sizes + resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, + }, + { + pageNumber: 1, + pageSize: 4, + }, + ); + expect(resp.statusCode).toEqual(200); + expectedResponse = allImportsExpected.slice(0, 4); + if (apiVersion === 'v2') { + expectedResponse = { + imports: expectedResponse, + page: 1, + size: 4, + totalCount: 6, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + + resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, + }, + { + pageNumber: 2, + pageSize: 4, + }, + ); + expect(resp.statusCode).toEqual(200); + expectedResponse = allImportsExpected.slice(4, 6); + if (apiVersion === 'v2') { + expectedResponse = { + imports: expectedResponse, + page: 2, + size: 4, + totalCount: 6, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + + // No data for this page + resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, }, - approvalTool: 'GIT', - status: 'WAIT_PR_APPROVAL', - github: { - pullRequest: { - number: 100, - url: 'https://github.com/my-org-2/my-repo-21/pull/100', + { + pageNumber: 3, + pageSize: 4, + }, + ); + expect(resp.statusCode).toEqual(200); + expectedResponse = []; + if (apiVersion === 'v2') { + expectedResponse = { + imports: expectedResponse, + page: 3, + size: 4, + totalCount: 6, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + }, + ); + + it.each([undefined, 'v1', 'v2'])( + 'should respect search and pagination when returning imports (API Version: %s)', + async apiVersionStr => { + jest + .spyOn(mockCatalogInfoGenerator, 'listCatalogUrlLocations') + .mockImplementation( + async ( + _config: Config, + search?: string | undefined, + _pageNumber?: number | undefined, + _pageSize?: number | undefined, + ) => { + const filteredLocations = search + ? locationUrls.filter(l => l.toLowerCase().includes(search)) + : locationUrls; + return { + targetUrls: filteredLocations, + totalCount: filteredLocations.length, + }; }, + ); + jest + .spyOn( + mockGithubApiService, + 'filterLocationsAccessibleFromIntegrations', + ) + .mockImplementation(async (locs: string[]) => { + const accessible = [ + // only repos that are accessible from the configured GH integrations + // are considered as valid Imports + 'https://github.com/my-org-1/my-repo-11/blob/main/catalog-info.yaml', // PR + 'https://github.com/my-user/my-repo-123/blob/main/catalog-info.yaml', // PR Error + 'https://github.com/my-org-2/my-repo-21/blob/master/catalog-info.yaml', // ADDED + 'https://github.com/my-org-2/my-repo-22/blob/master/catalog-info.yaml', // no PR => null status + 'https://github.com/my-org-3/my-repo-31/blob/main/catalog-info.yaml', // ADDED + 'https://github.com/my-org-3/my-repo-32/blob/dev/catalog-info.yaml', // PR + ]; + return locs.filter(loc => accessible.includes(loc)); + }); + jest + .spyOn(mockCatalogInfoGenerator, 'findLocationEntitiesByTargetUrl') + .mockResolvedValue([]); + + const apiVersion = apiVersionStr as + | Paths.FindAllImports.Parameters.ApiVersion + | undefined; + let resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, }, - }, - ]); - }); + { + search: 'lorem ipsum dolor sit amet should not return any data', + }, + ); + expect(resp.statusCode).toEqual(200); + let expectedResponse: any = []; + if (apiVersion === 'v2') { + expectedResponse = { + imports: expectedResponse, + page: 1, + size: 20, + totalCount: 0, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + + resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, + }, + { + search: 'my-repo-2', + }, + ); + expect(resp.statusCode).toEqual(200); + const allImportsExpected = [ + { + id: 'https://github.com/my-org-2/my-repo-21', + repository: { + url: 'https://github.com/my-org-2/my-repo-21', + name: 'my-repo-21', + organization: 'my-org-2', + id: 'my-org-2/my-repo-21', + defaultBranch: 'master', + }, + approvalTool: 'GIT', + status: 'ADDED', + }, + { + id: 'https://github.com/my-org-2/my-repo-22', + repository: { + url: 'https://github.com/my-org-2/my-repo-22', + name: 'my-repo-22', + organization: 'my-org-2', + id: 'my-org-2/my-repo-22', + defaultBranch: 'master', + }, + approvalTool: 'GIT', + status: null, + }, + ]; + expectedResponse = allImportsExpected; + if (apiVersion === 'v2') { + expectedResponse = { + imports: expectedResponse, + page: 1, + size: 20, + totalCount: 2, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + + // Request different pages and sizes + resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, + }, + { + search: 'my-repo-2', + pageNumber: 1, + pageSize: 1, + }, + ); + expect(resp.statusCode).toEqual(200); + expectedResponse = allImportsExpected.slice(0, 1); + if (apiVersion === 'v2') { + expectedResponse = { + imports: expectedResponse, + page: 1, + size: 1, + totalCount: 2, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + + resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, + }, + { + search: 'my-repo-2', + pageNumber: 2, + pageSize: 1, + }, + ); + expect(resp.statusCode).toEqual(200); + expectedResponse = allImportsExpected.slice(1, 2); + if (apiVersion === 'v2') { + expectedResponse = { + imports: expectedResponse, + page: 2, + size: 1, + totalCount: 2, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + + // No data for this page + resp = await findAllImports( + logger, + config, + mockGithubApiService, + mockCatalogInfoGenerator, + { + apiVersion, + }, + { + search: 'my-repo-2', + pageNumber: 3, + pageSize: 1, + }, + ); + expect(resp.statusCode).toEqual(200); + expectedResponse = []; + if (apiVersion === 'v2') { + expectedResponse = { + imports: expectedResponse, + page: 3, + size: 1, + totalCount: 2, + }; + } + expect(resp.responseBody).toEqual(expectedResponse); + }, + ); }); describe('deleteImportByRepo', () => { @@ -314,12 +619,15 @@ describe('bulkimports.ts tests', () => { mockCatalogInfoGenerator, 'listCatalogUrlLocationsByIdFromLocationsEndpoint', ) - .mockResolvedValue([ - { - id: 'location-id-11', - target: `${repoUrl}/blob/${defaultBranch}/catalog-info.yaml`, - }, - ]); + .mockResolvedValue({ + locations: [ + { + id: 'location-id-11', + target: `${repoUrl}/blob/${defaultBranch}/catalog-info.yaml`, + }, + ], + totalCount: 1, + }); jest .spyOn(mockCatalogInfoGenerator, 'deleteCatalogLocationById') .mockResolvedValue(); @@ -366,12 +674,15 @@ describe('bulkimports.ts tests', () => { mockCatalogInfoGenerator, 'listCatalogUrlLocationsByIdFromLocationsEndpoint', ) - .mockResolvedValue([ - { - id: 'location-id-12', - target: `${repoUrl}/blob/${defaultBranch}/catalog-info.yaml`, - }, - ]); + .mockResolvedValue({ + locations: [ + { + id: 'location-id-12', + target: `${repoUrl}/blob/${defaultBranch}/catalog-info.yaml`, + }, + ], + totalCount: 1, + }); jest .spyOn(mockCatalogInfoGenerator, 'deleteCatalogLocationById') .mockResolvedValue(); diff --git a/plugins/bulk-import-backend/src/service/handlers/bulkImports.ts b/plugins/bulk-import-backend/src/service/handlers/bulkImports.ts index 570055f387..d40d6b564d 100644 --- a/plugins/bulk-import-backend/src/service/handlers/bulkImports.ts +++ b/plugins/bulk-import-backend/src/service/handlers/bulkImports.ts @@ -42,25 +42,43 @@ type CreateImportDryRunStatus = | 'CODEOWNERS_FILE_NOT_FOUND_IN_REPO' | 'REPO_EMPTY'; +type FindAllImportsResponse = + | Components.Schemas.Import[] + | Components.Schemas.ImportJobListV2; + export async function findAllImports( logger: LoggerService, config: Config, githubApiService: GithubApiService, catalogInfoGenerator: CatalogInfoGenerator, - search?: string, - pageNumber: number = DefaultPageNumber, - pageSize: number = DefaultPageSize, -): Promise> { - logger.debug('Getting all bulk import jobs..'); + requestHeaders?: { + apiVersion?: Paths.FindAllImports.Parameters.ApiVersion; + }, + queryParams?: { + search?: string; + pageNumber?: number; + pageSize?: number; + }, +): Promise> { + const apiVersion = requestHeaders?.apiVersion ?? 'v1'; + const search = queryParams?.search; + const pageNumber = queryParams?.pageNumber ?? DefaultPageNumber; + const pageSize = queryParams?.pageSize ?? DefaultPageSize; + + logger.debug( + `Getting all bulk import jobs (apiVersion=${apiVersion}, search=${search}, page=${pageNumber}, size=${pageSize})..`, + ); const catalogFilename = getCatalogFilename(config); - const allLocations = await catalogInfoGenerator.listCatalogUrlLocations( - config, - search, - pageNumber, - pageSize, - ); + const allLocations = ( + await catalogInfoGenerator.listCatalogUrlLocations( + config, + search, + pageNumber, + pageSize, + ) + ).targetUrls; // resolve default branches for each unique repo URL from GH, // because we cannot easily determine that from the location target URL. @@ -126,9 +144,21 @@ export async function findAllImports( } return a.repository.name.localeCompare(b.repository.name); }); + const paginated = paginateArray(imports, pageNumber, pageSize); + if (apiVersion === 'v1') { + return { + statusCode: 200, + responseBody: paginated.result, + }; + } return { statusCode: 200, - responseBody: paginateArray(imports, pageNumber, pageSize).result, + responseBody: { + imports: paginated.result, + totalCount: paginated.totalCount, + page: pageNumber, + size: pageSize, + }, }; } @@ -594,8 +624,9 @@ export async function findImportStatusByRepo( includeCatalogInfoContent, }); if (!openImportPr.prUrl) { - const catalogLocations = - await catalogInfoGenerator.listCatalogUrlLocations(config); + const catalogLocations = ( + await catalogInfoGenerator.listCatalogUrlLocations(config) + ).targetUrls; const catalogUrl = catalogInfoGenerator.getCatalogUrl( config, repoUrl, @@ -708,7 +739,9 @@ export async function deleteImportByRepo( }; const locationId = findLocationFrom( - await catalogInfoGenerator.listCatalogUrlLocationsByIdFromLocationsEndpoint(), + ( + await catalogInfoGenerator.listCatalogUrlLocationsByIdFromLocationsEndpoint() + ).locations, ); if (locationId) { await catalogInfoGenerator.deleteCatalogLocationById(locationId); diff --git a/plugins/bulk-import-backend/src/service/handlers/repositories.ts b/plugins/bulk-import-backend/src/service/handlers/repositories.ts index 811b5defea..a39f6a6d3d 100644 --- a/plugins/bulk-import-backend/src/service/handlers/repositories.ts +++ b/plugins/bulk-import-backend/src/service/handlers/repositories.ts @@ -119,7 +119,7 @@ async function formatResponse( } const catalogLocations = checkStatus - ? await catalogInfoGenerator.listCatalogUrlLocations(config) + ? (await catalogInfoGenerator.listCatalogUrlLocations(config)).targetUrls : []; const repoList: Components.Schemas.Repository[] = []; if (allReposAccessible.repositories) { diff --git a/plugins/bulk-import-backend/src/service/router.test.ts b/plugins/bulk-import-backend/src/service/router.test.ts index 26b6e03bda..aa64d98fa4 100644 --- a/plugins/bulk-import-backend/src/service/router.test.ts +++ b/plugins/bulk-import-backend/src/service/router.test.ts @@ -596,121 +596,149 @@ describe('bulk-import router tests', () => { }); describe('GET /imports', () => { - it('returns 200 with empty list when there is nothing in catalog yet and no open PR for each repo', async () => { - const backendServer = await startBackendServer(AuthorizeResult.ALLOW, { - catalog: { locations: [] }, - }); - server.use( - rest.get( - `http://localhost:${backendServer.port()}/api/catalog/locations`, - (_, res, ctx) => res(ctx.status(200), ctx.json([])), - ), - ); - mockCatalogClient.queryEntities = jest - .fn() - .mockResolvedValue({ items: [] }); + it.each([undefined, 'v1', 'v2'])( + 'returns 200 with empty list when there is nothing in catalog yet and no open PR for each repo (API Version: %s)', + async apiVersion => { + const backendServer = await startBackendServer(AuthorizeResult.ALLOW, { + catalog: { locations: [] }, + }); + server.use( + rest.get( + `http://localhost:${backendServer.port()}/api/catalog/locations`, + (_, res, ctx) => res(ctx.status(200), ctx.json([])), + ), + ); + mockCatalogClient.queryEntities = jest + .fn() + .mockResolvedValue({ items: [] }); - const response = await request(backendServer).get( - '/api/bulk-import/imports', - ); + let req = request(backendServer).get('/api/bulk-import/imports'); + if (apiVersion) { + req = req.set('api-version', apiVersion); + } + const response = await req; - expect(response.status).toEqual(200); - expect(response.body).toEqual([]); - }); + expect(response.status).toEqual(200); + let expectedRespBody: any = []; + if (apiVersion === 'v2') { + expectedRespBody = { + imports: expectedRespBody, + page: 1, + size: 20, + totalCount: 0, + }; + } + expect(response.body).toEqual(expectedRespBody); + }, + ); - it('returns 200 with appropriate import status (with data coming from the repos and data coming from the app-config files)', async () => { - const backendServer = await startBackendServer(AuthorizeResult.ALLOW); - server.use( - rest.get( - `http://localhost:${backendServer.port()}/api/catalog/locations`, - (_, res, ctx) => - res( - ctx.status(200), - ctx.json(loadTestFixture('catalog/locations.json')), - ), - ), - ); - mockCatalogClient.queryEntities = jest - .fn() - .mockImplementation( - async ( - _request?: QueryEntitiesRequest, - _options?: CatalogRequestOptions, - ): Promise => { - return { - items: [ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - name: `generated-from-tests-${Math.floor(Math.random() * 100 + 1)}`, - namespace: 'default', - }, - }, - ], - totalItems: 1, - pageInfo: {}, - }; - }, + it.each([undefined, 'v1', 'v2'])( + 'returns 200 with appropriate import status (with data coming from the repos and data coming from the app-config files) (API Version: %s)', + async apiVersion => { + const backendServer = await startBackendServer(AuthorizeResult.ALLOW); + server.use( + rest.get( + `http://localhost:${backendServer.port()}/api/catalog/locations`, + (_, res, ctx) => + res( + ctx.status(200), + ctx.json(loadTestFixture('catalog/locations.json')), + ), + ), ); + mockCatalogClient.queryEntities = jest + .fn() + .mockImplementation( + async ( + _request?: QueryEntitiesRequest, + _options?: CatalogRequestOptions, + ): Promise => { + return { + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: `generated-from-tests-${Math.floor(Math.random() * 100 + 1)}`, + namespace: 'default', + }, + }, + ], + totalItems: 1, + pageInfo: {}, + }; + }, + ); - const response = await request(backendServer).get( - '/api/bulk-import/imports', - ); + let req = request(backendServer).get('/api/bulk-import/imports'); + if (apiVersion) { + req = req.set('api-version', apiVersion); + } + const response = await req; - expect(response.status).toEqual(200); - expect(response.body).toEqual([ - { - approvalTool: 'GIT', - id: 'https://github.com/octocat/my-awesome-repo', - lastUpdate: '2011-01-26T19:14:43Z', - repository: { - defaultBranch: 'dev', - id: 'octocat/my-awesome-repo', - name: 'my-awesome-repo', - organization: 'octocat', - url: 'https://github.com/octocat/my-awesome-repo', - }, - status: null, - }, - { - approvalTool: 'GIT', - id: 'https://github.com/my-org-1/my-repo-with-existing-catalog-info-in-default-branch', - lastUpdate: '2011-01-26T19:14:43Z', - repository: { - defaultBranch: 'main', - id: 'my-org-1/my-repo-with-existing-catalog-info-in-default-branch', - name: 'my-repo-with-existing-catalog-info-in-default-branch', - organization: 'my-org-1', - url: 'https://github.com/my-org-1/my-repo-with-existing-catalog-info-in-default-branch', + expect(response.status).toEqual(200); + let expectedRespBody: any = [ + { + approvalTool: 'GIT', + id: 'https://github.com/octocat/my-awesome-repo', + lastUpdate: '2011-01-26T19:14:43Z', + repository: { + defaultBranch: 'dev', + id: 'octocat/my-awesome-repo', + name: 'my-awesome-repo', + organization: 'octocat', + url: 'https://github.com/octocat/my-awesome-repo', + }, + status: null, }, - status: 'ADDED', - }, - { - approvalTool: 'GIT', - github: { - pullRequest: { - body: 'Onboarding this repository into Red Hat Developer Hub.', - number: 1347, - title: 'Add catalog-info.yaml', - url: 'https://github.com/my-org-1/my-repo-with-no-catalog-info-in-default-branch-and-import-pr/pull/1347', + { + approvalTool: 'GIT', + id: 'https://github.com/my-org-1/my-repo-with-existing-catalog-info-in-default-branch', + lastUpdate: '2011-01-26T19:14:43Z', + repository: { + defaultBranch: 'main', + id: 'my-org-1/my-repo-with-existing-catalog-info-in-default-branch', + name: 'my-repo-with-existing-catalog-info-in-default-branch', + organization: 'my-org-1', + url: 'https://github.com/my-org-1/my-repo-with-existing-catalog-info-in-default-branch', }, + status: 'ADDED', }, - id: 'https://github.com/my-org-1/my-repo-with-no-catalog-info-in-default-branch-and-import-pr', - lastUpdate: '2011-01-26T19:01:12Z', - repository: { - defaultBranch: 'main', - id: 'my-org-1/my-repo-with-no-catalog-info-in-default-branch-and-import-pr', - name: 'my-repo-with-no-catalog-info-in-default-branch-and-import-pr', - organization: 'my-org-1', - url: 'https://github.com/my-org-1/my-repo-with-no-catalog-info-in-default-branch-and-import-pr', + { + approvalTool: 'GIT', + github: { + pullRequest: { + body: 'Onboarding this repository into Red Hat Developer Hub.', + number: 1347, + title: 'Add catalog-info.yaml', + url: 'https://github.com/my-org-1/my-repo-with-no-catalog-info-in-default-branch-and-import-pr/pull/1347', + }, + }, + id: 'https://github.com/my-org-1/my-repo-with-no-catalog-info-in-default-branch-and-import-pr', + lastUpdate: '2011-01-26T19:01:12Z', + repository: { + defaultBranch: 'main', + id: 'my-org-1/my-repo-with-no-catalog-info-in-default-branch-and-import-pr', + name: 'my-repo-with-no-catalog-info-in-default-branch-and-import-pr', + organization: 'my-org-1', + url: 'https://github.com/my-org-1/my-repo-with-no-catalog-info-in-default-branch-and-import-pr', + }, + status: 'WAIT_PR_APPROVAL', }, - status: 'WAIT_PR_APPROVAL', - }, - ]); - // Location entity refresh triggered (on each 'ADDED' repo) - expect(mockCatalogClient.refreshEntity).toHaveBeenCalledTimes(1); - }); + ]; + if (apiVersion === 'v2') { + expectedRespBody = { + imports: expectedRespBody, + page: 1, + size: 20, + totalCount: 3, + }; + } + expect(response.body).toEqual(expectedRespBody); + // Location entity refresh triggered (on each 'ADDED' repo) + expect(mockCatalogClient.refreshEntity).toHaveBeenCalledTimes(1); + }, + ); }); describe('POST /imports', () => { diff --git a/plugins/bulk-import-backend/src/service/router.ts b/plugins/bulk-import-backend/src/service/router.ts index 6702fea507..7d241be5bd 100644 --- a/plugins/bulk-import-backend/src/service/router.ts +++ b/plugins/bulk-import-backend/src/service/router.ts @@ -227,20 +227,38 @@ export async function createRouter( api.register( 'findAllImports', async (c: Context, _req: Request, res: Response) => { + const h: Paths.FindAllImports.HeaderParameters = { + ...c.request.headers, + }; + const apiVersion = h['api-version']; const q: Paths.FindAllImports.QueryParameters = { ...c.request.query, }; // we need to convert strings to real types due to open PR https://github.com/openapistack/openapi-backend/pull/571 - q.pagePerIntegration = stringToNumber(q.pagePerIntegration); - q.sizePerIntegration = stringToNumber(q.sizePerIntegration); + let page: number | undefined; + let size: number | undefined; + if (apiVersion === undefined || apiVersion === 'v1') { + // pagePerIntegration and sizePerIntegration deprecated in v1. 'page' and 'size' take precedence. + page = stringToNumber(q.page || q.pagePerIntegration); + size = stringToNumber(q.size || q.sizePerIntegration); + } else { + // pagePerIntegration and sizePerIntegration removed in v2+ and replaced by 'page' and 'size'. + page = stringToNumber(q.page); + size = stringToNumber(q.size); + } const response = await findAllImports( logger, config, githubApiService, catalogInfoGenerator, - q.search, - q.pagePerIntegration, - q.sizePerIntegration, + { + apiVersion, + }, + { + search: q.search, + pageNumber: page, + pageSize: size, + }, ); return res.status(response.statusCode).json(response.responseBody); },