diff --git a/package-lock.json b/package-lock.json index d9e71165..369019cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -96,12 +96,12 @@ } }, "node_modules/@adobe/helix-docx2md": { - "version": "1.6.12", - "resolved": "https://registry.npmjs.org/@adobe/helix-docx2md/-/helix-docx2md-1.6.12.tgz", - "integrity": "sha512-JvuiqIfUNpMc35WJtr5I4/9mQX4v/tPbZzEhtONQXy6Knovae5VvAIVEul4dj4gXyowKAOD0pU6Ofkn8ZGG7vQ==", + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/@adobe/helix-docx2md/-/helix-docx2md-1.6.11.tgz", + "integrity": "sha512-8EJVE2+sK8PFhZbpdgwTV+vzTZ5+PXYbFzj3f/AEubvo9yN20SV0vqX55cw9Kislqkq9v54aqffLRG1uHaDGDQ==", "license": "Apache-2.0", "dependencies": { - "@adobe/helix-markdown-support": "7.1.9", + "@adobe/helix-markdown-support": "7.1.8", "@adobe/helix-shared-process-queue": "3.0.4", "@adobe/mammoth": "1.7.1-bleeding.2", "@adobe/mdast-util-gridtables": "4.0.8", @@ -120,12 +120,12 @@ } }, "node_modules/@adobe/helix-markdown-support": { - "version": "7.1.9", - "resolved": "https://registry.npmjs.org/@adobe/helix-markdown-support/-/helix-markdown-support-7.1.9.tgz", - "integrity": "sha512-twtcoGUw5EvnLTaWIIv7m3xgy0XvwMyOSZTdWq19efK0W6LzKlsbd2tFP0QmLl6+3/ddzjssDNVnc937dLQI4w==", + "version": "7.1.8", + "resolved": "https://registry.npmjs.org/@adobe/helix-markdown-support/-/helix-markdown-support-7.1.8.tgz", + "integrity": "sha512-n+mtjrqqERtT7dJSleBbr/ZjccJxn3t4U8HkRuOpOjSObHi73CgLRyu/dtxmGtobmItIpkUBD6tTvzrHrEjXcQ==", "license": "Apache-2.0", "dependencies": { - "hast-util-to-html": "9.0.4", + "hast-util-to-html": "9.0.3", "js-yaml": "4.1.0", "mdast-util-gfm-footnote": "2.0.0", "mdast-util-gfm-strikethrough": "2.0.0", @@ -152,14 +152,14 @@ } }, "node_modules/@adobe/helix-md2docx": { - "version": "2.1.92", - "resolved": "https://registry.npmjs.org/@adobe/helix-md2docx/-/helix-md2docx-2.1.92.tgz", - "integrity": "sha512-Ri7t4X9UxNM7ArGI44nyzVi7D/PJejoRtbsDrzXLOjYNymDEsWjkIeH3rW7qDPlmTrr2hbBuv+1tGr/LhfzO1A==", + "version": "2.1.90", + "resolved": "https://registry.npmjs.org/@adobe/helix-md2docx/-/helix-md2docx-2.1.90.tgz", + "integrity": "sha512-NUXFFeRK/DTTovt7TF/FOYfMBIMxkQHT3bqInIfLMqTOu3ga+E96T942zwSoUcwhzCbf0jF/N3+WeLJfr2+RJg==", "license": "Apache-2.0", "dependencies": { "@adobe/fetch": "4.1.11", - "@adobe/helix-docx2md": "1.6.12", - "@adobe/helix-markdown-support": "7.1.9", + "@adobe/helix-docx2md": "1.6.11", + "@adobe/helix-markdown-support": "7.1.8", "@adobe/helix-shared-process-queue": "3.0.4", "@adobe/remark-gridtables": "3.0.8", "docx": "9.1.0", @@ -272,21 +272,21 @@ } }, "node_modules/@adobe/rum-distiller": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/@adobe/rum-distiller/-/rum-distiller-1.13.1.tgz", - "integrity": "sha512-NNzeRwH6B+XrJi4Pjz6ITjXYI8zh2JKC6zc+pkfqORnOv2WgOmxAJEM+qF1NOMekxY/7a2XqS1Z2avEons7bkg==", + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/@adobe/rum-distiller/-/rum-distiller-1.13.0.tgz", + "integrity": "sha512-/F84FE0HmX1pq/I6VCR9bd5WBokwzz+SKdGMvdRo73Z8AwPhXStWZpjkaIK48p1fHHduS5TMBp+Hn94HB/QEOA==", "license": "Apache-2.0" }, "node_modules/@adobe/spacecat-helix-content-sdk": { - "version": "1.3.15", - "resolved": "https://registry.npmjs.org/@adobe/spacecat-helix-content-sdk/-/spacecat-helix-content-sdk-1.3.15.tgz", - "integrity": "sha512-BXfg0dUyYvDcWyH3VbUTQ7rOJAup96GchxZ+5QBFf+zkcGWkapKT/5guMDjQ7P5rKiyD8n++Ox9fQruxemQrwQ==", + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/@adobe/spacecat-helix-content-sdk/-/spacecat-helix-content-sdk-1.3.14.tgz", + "integrity": "sha512-1qiORhWPN9lgHSvhZc/NyRhyI7j35SHuZglwuIHLsPa0zfPiPcVrXHllZUvDkSeiDVdVTSAGbyZifCizVK8ZhA==", "license": "Apache-2.0", "dependencies": { "@adobe/fetch": "4.1.11", - "@adobe/helix-docx2md": "1.6.12", + "@adobe/helix-docx2md": "1.6.11", "@adobe/helix-markdown-support": "7.1.9", - "@adobe/helix-md2docx": "2.1.92", + "@adobe/helix-md2docx": "2.1.90", "@adobe/mdast-util-gridtables": "4.0.8", "@azure/msal-node": "2.16.2", "@googleapis/docs": "3.3.0", @@ -304,6 +304,61 @@ "unist-util-visit": "5.0.0" } }, + "node_modules/@adobe/spacecat-helix-content-sdk/node_modules/@adobe/helix-markdown-support": { + "version": "7.1.9", + "resolved": "https://registry.npmjs.org/@adobe/helix-markdown-support/-/helix-markdown-support-7.1.9.tgz", + "integrity": "sha512-twtcoGUw5EvnLTaWIIv7m3xgy0XvwMyOSZTdWq19efK0W6LzKlsbd2tFP0QmLl6+3/ddzjssDNVnc937dLQI4w==", + "license": "Apache-2.0", + "dependencies": { + "hast-util-to-html": "9.0.4", + "js-yaml": "4.1.0", + "mdast-util-gfm-footnote": "2.0.0", + "mdast-util-gfm-strikethrough": "2.0.0", + "mdast-util-gfm-table": "2.0.0", + "mdast-util-gfm-task-list-item": "2.0.0", + "mdast-util-phrasing": "4.1.0", + "mdast-util-to-hast": "13.2.0", + "micromark-extension-gfm-footnote": "2.1.0", + "micromark-extension-gfm-strikethrough": "2.1.0", + "micromark-extension-gfm-table": "2.1.0", + "micromark-extension-gfm-tagfilter": "2.0.0", + "micromark-extension-gfm-task-list-item": "2.1.0", + "micromark-util-character": "2.1.1", + "micromark-util-combine-extensions": "2.0.1", + "micromark-util-symbol": "2.0.1", + "unist-util-find": "3.0.0", + "unist-util-visit": "5.0.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "unified": "11.x" + } + }, + "node_modules/@adobe/spacecat-helix-content-sdk/node_modules/hast-util-to-html": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.4.tgz", + "integrity": "sha512-wxQzXtdbhiwGAUKrnQJXlOPmHnEehzphwkK7aluUPQ+lEc1xefC8pblMgpp2w5ldBTEfveRIrADcrhGIWrlTDA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/@adobe/spacecat-shared-ahrefs-client": { "resolved": "packages/spacecat-shared-ahrefs-client", "link": true @@ -10422,9 +10477,9 @@ } }, "node_modules/hast-util-to-html": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.4.tgz", - "integrity": "sha512-wxQzXtdbhiwGAUKrnQJXlOPmHnEehzphwkK7aluUPQ+lEc1xefC8pblMgpp2w5ldBTEfveRIrADcrhGIWrlTDA==", + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.3.tgz", + "integrity": "sha512-M17uBDzMJ9RPCqLMO92gNNUDuBSq10a25SDBI08iCCxmorf4Yy6sYHK57n9WAbRAAaU+DuR4W6GN9K4DFZesYg==", "license": "MIT", "dependencies": { "@types/hast": "^3.0.0", @@ -21775,11 +21830,11 @@ }, "packages/spacecat-shared-content-client": { "name": "@adobe/spacecat-shared-content-client", - "version": "1.2.16", + "version": "1.2.15", "license": "Apache-2.0", "dependencies": { "@adobe/helix-universal": "5.0.8", - "@adobe/spacecat-helix-content-sdk": "1.3.15", + "@adobe/spacecat-helix-content-sdk": "1.3.14", "@adobe/spacecat-shared-data-access": "1.50.0", "@adobe/spacecat-shared-utils": "1.22.4", "graph-data-structure": "4.3.0" @@ -29260,7 +29315,7 @@ "@adobe/fetch": "4.1.11", "@adobe/helix-shared-wrap": "2.0.2", "@adobe/helix-universal": "5.0.8", - "@adobe/rum-distiller": "1.13.1", + "@adobe/rum-distiller": "1.13.0", "@adobe/spacecat-shared-utils": "1.22.4", "aws4": "1.13.2", "urijs": "^1.19.11" diff --git a/packages/spacecat-shared-data-access/migration_v2.sh b/packages/spacecat-shared-data-access/migration_v2.sh new file mode 100644 index 00000000..e20f00ec --- /dev/null +++ b/packages/spacecat-shared-data-access/migration_v2.sh @@ -0,0 +1,350 @@ +#!/bin/bash + +# Define AWS CLI command with local DynamoDB endpoint +AWS_LOCAL_CMD="aws dynamodb --region test-region --endpoint-url http://localhost:8000" +REGION="us-east-1" +AWS_CMD="aws dynamodb --region $REGION" + +# Define table names +SITE_TABLE="spacecat-services-sites-dev" +ORGANIZATION_TABLE="spacecat-services-organizations-dev" +CONFIGURATION_TABLE="spacecat-services-configurations-dev" +DATA_TABLE="spacecat-services-data-dev" +EXPERIMENT_TABLE="spacecat-services-experiments-dev" +SITE_CANDIDATE_TABLE="spacecat-services-site-candidates-dev" +TOP_PAGES_TABLE="spacecat-services-site-top-pages-dev" +KEY_EVENTS_TABLE="spacecat-services-key-events-dev" + +# Fetch all sites +SITES=$($AWS_CMD scan --table-name $SITE_TABLE) +ORGANIZATIONS=$($AWS_CMD scan --table-name $ORGANIZATION_TABLE) +CONFIGURATIONS=$($AWS_CMD scan --table-name $CONFIGURATION_TABLE) +EXPERIMENTS=$($AWS_CMD scan --table-name $EXPERIMENT_TABLE) +SITE_CANDIDATES=$($AWS_CMD scan --table-name $SITE_CANDIDATE_TABLE) +TOP_PAGES=$($AWS_CMD scan --table-name $TOP_PAGES_TABLE) +KEY_EVENTS=$($AWS_CMD scan --table-name $KEY_EVENTS_TABLE) + +# Migrate each site +echo "$SITES" | jq -c '.Items[]' | while read -r site; do + SITE_ID=$(echo $site | jq -r '.id.S') + SITE_PK="\$spacecat#siteid_$SITE_ID" + SITE_SK="\$site_1" + BASE_URL=$(echo $site | jq -r '.baseURL.S') + DELIVERY_TYPE=$(echo $site | jq -r '.deliveryType.S') + GITHUB_URL=$(echo $site | jq -r '.gitHubURL.S') + ORG_ID=$(echo $site | jq -r '.organizationId.S') + IS_LIVE=$(echo $site | jq -r '.isLive.BOOL // false') + IS_LIVE_TOGGLED_AT=$(echo $site | jq -r '.isLiveToggledAt.S // empty') + CREATED_AT=$(echo $site | jq -r '.createdAt.S') + UPDATED_AT=$(echo $site | jq -r '.updatedAt.S') + CONFIG=$(echo $site | jq -r '.config // {"M": {}}') + HLX_CONFIG=$(echo $site | jq -r '.hlxConfig // {"M": {}}') + SITE_GSI1PK="all_sites" + SITE_GSI1SK="$SITE_SK#baseurl_${BASE_URL}" + SITE_GSI2PK="\$spacecat#organizationId_$ORG_ID" + SITE_GSI2SK="$SITE_SK#updatedat_$UPDATED_AT" + SITE_GSI3PK="\$spacecat#deliverytype_$DELIVERY_TYPE" + SITE_GSI3SK="$SITE_GSI2SK" + + MIGRATED_SITE=$(cat < { * @param {Logger} log logger * @returns {object} data access object */ -export const createDataAccess = (config, log = console, client = undefined) => { - const dynamoClient = createClient(log, client); +export const createDataAccess = async (config, log = console, client = undefined) => { + const dynamoClient = await createClient(log, client); const auditFuncs = auditFunctions(dynamoClient, config, log); const keyEventFuncs = keyEventFunctions(dynamoClient, config, log); diff --git a/packages/spacecat-shared-data-access/test/it/migration.test.js b/packages/spacecat-shared-data-access/test/it/migration.test.js new file mode 100644 index 00000000..36d20a42 --- /dev/null +++ b/packages/spacecat-shared-data-access/test/it/migration.test.js @@ -0,0 +1,106 @@ +/* + * Copyright 2024 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +/* eslint-disable no-console */ + +import { expect } from 'chai'; +import { getDataAccess } from './util/db.js'; + +// TODO WIP fix errors and remove skip and run after migrate_v2.sh has finished running +describe('Migration Tests', () => { + let migratedDataAccess; + let originalDataAccess; + const originalCounts = {}; + const entities = ['Site', 'Organization']; + + before(async () => { + // Get original counts + migratedDataAccess = await getDataAccess({}, true); + originalDataAccess = await getDataAccess({ + region: process.env.AWS_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + sessionToken: process.env.AWS_SESSION_TOKEN, + }, + }, true); + try { + const sites = await originalDataAccess.getSites(); + originalCounts.Site = sites.length; + const organizations = await originalDataAccess.getOrganizations(); + originalCounts.Organization = organizations.length; + } catch (error) { + console.log('Error getting original counts', error.message); + } + }); + + entities.forEach((entity) => { + it(`should have the same number of ${entity} entities after migration`, async () => { + const migratedCount = (await migratedDataAccess[entity].all()).length; + expect(migratedCount).to.equal(originalCounts[entity]); + }).timeout(300000); + + it(`should correctly retrieve ${entity} by ID after migration`, async () => { + const allEntities = await migratedDataAccess[entity].all(); + const randomEntity = allEntities[Math.floor(Math.random() * allEntities.length)]; + const retrievedEntity = await migratedDataAccess[entity].findById(randomEntity.getId()); + expect(retrievedEntity).to.deep.equal(randomEntity); + }).timeout(300000); + }); + + it('should maintain relationships between entities after migration', async () => { + const sites = (await migratedDataAccess.Site.all()); + for (const site of sites) { + // eslint-disable-next-line no-await-in-loop + const organizationId = await site.getOrganizationId(); + if (organizationId !== 'default') { + // eslint-disable-next-line no-await-in-loop + const organization = await migratedDataAccess.Organization.findById(organizationId); + expect(organization).to.not.be.null; + expect(organization.getId()).to.equal(organizationId); + } + } + }).timeout(300000); + + xit('should correctly update an entity after migration', async () => { + const site = (await migratedDataAccess.Site.getAll())[0]; + const originalName = site.getName(); + const newName = 'Updated Site Name'; + + site.setName(newName); + await site.save(); + + const updatedSite = await migratedDataAccess.Site.findById(site.getId()); + expect(updatedSite.getName()).to.equal(newName); + + // Revert the change + site.setName(originalName); + await site.save(); + }); + + xit('should correctly create and delete an entity after migration', async () => { + const newSite = await migratedDataAccess.Site.create({ + baseURL: 'https://example.com', + organizationId: (await migratedDataAccess.Organization.getAll())[0].getId(), + // Add other required fields + }); + + const retrievedSite = await migratedDataAccess.Site.findById(newSite.getId()); + expect(retrievedSite).to.deep.equal(newSite); + + await newSite.remove(); + + const deletedSite = await migratedDataAccess.Site.findById(newSite.getId()); + expect(deletedSite).to.be.null; + }); +}); diff --git a/packages/spacecat-shared-data-access/test/it/util/db.js b/packages/spacecat-shared-data-access/test/it/util/db.js index 8d1a6450..5af212fa 100755 --- a/packages/spacecat-shared-data-access/test/it/util/db.js +++ b/packages/spacecat-shared-data-access/test/it/util/db.js @@ -49,6 +49,40 @@ export const TEST_DA_CONFIG = { tableNameSpacecatData: 'spacecat-data', }; +export const TEST_DA_MIGRATION_CONFIG = { + indexNameAllImportJobsByDateRange: 'spacecat-services-all-import-jobs-by-date-range', + indexNameAllImportJobsByStatus: 'spacecat-services-all-import-jobs-by-status', + indexNameAllKeyEventsBySiteId: 'spacecat-services-key-events-by-site-id-dev', + indexNameAllLatestAuditScores: 'spacecat-services-all-latest-audit-scores', + indexNameAllOrganizations: 'spacecat-services-all-organizations-dev', + indexNameAllOrganizationsByImsOrgId: 'spacecat-services-all-organizations-by-ims-org-id-dev', + indexNameAllSites: 'spacecat-services-all-sites-dev', + indexNameAllSitesByDeliveryType: 'spacecat-services-all-sites-by-delivery-type-dev', + indexNameAllSitesOrganizations: 'spacecat-services-all-sites-organizations-dev', + indexNameApiKeyByHashedApiKey: 'spacecat-services-api-key-by-hashed-api-key', + indexNameApiKeyByImsUserIdAndImsOrgId: 'spacecat-services-api-key-by-ims-user-id-and-ims-org-id', + indexNameImportUrlsByJobIdAndStatus: 'spacecat-services-all-import-urls-by-job-id-and-status', + pkAllConfigurations: 'ALL_CONFIGURATIONS', + pkAllImportJobs: 'ALL_IMPORT_JOBS', + pkAllLatestAudits: 'ALL_LATEST_AUDITS', + pkAllOrganizations: 'ALL_ORGANIZATIONS', + pkAllSites: 'ALL_SITES', + tableNameApiKeys: 'spacecat-services-api-keys-dev', + tableNameAudits: 'spacecat-services-audits-dev', + tableNameConfigurations: 'spacecat-services-configurations-dev', + tableNameData: 'spacecat-services-data-dev', + tableNameExperiments: 'spacecat-services-experiments-dev', + tableNameImportJobs: 'spacecat-services-import-jobs-dev', + tableNameImportUrls: 'spacecat-services-import-urls-dev', + tableNameKeyEvents: 'spacecat-services-key-events-dev', + tableNameLatestAudits: 'spacecat-services-latest-audits-dev', + tableNameOrganizations: 'spacecat-services-organizations-dev', + tableNameSiteCandidates: 'spacecat-services-site-candidates-dev', + tableNameSiteTopPages: 'spacecat-services-site-top-pages-dev', + tableNameSites: 'spacecat-services-sites-dev', + tableNameSpacecatData: 'spacecat-services-data-dev', +}; + let docClient = null; const getDynamoClients = (config = {}) => { @@ -70,9 +104,11 @@ const getDynamoClients = (config = {}) => { return { dbClient, docClient }; }; -export const getDataAccess = (config) => { +export const getDataAccess = async (config, isMigration = false) => { const { dbClient } = getDynamoClients(config); - return createDataAccess(TEST_DA_CONFIG, console, dbClient); + return createDataAccess(isMigration + ? TEST_DA_MIGRATION_CONFIG + : TEST_DA_CONFIG, console, dbClient); }; export { getDynamoClients }; diff --git a/packages/spacecat-shared-dynamo/src/index.js b/packages/spacecat-shared-dynamo/src/index.js index a88d0f5e..160192f9 100644 --- a/packages/spacecat-shared-dynamo/src/index.js +++ b/packages/spacecat-shared-dynamo/src/index.js @@ -28,7 +28,7 @@ import removeItem from './modules/removeItem.js'; * @param {DynamoDBDocument} docClient - The AWS SDK DynamoDB Document client instance. * @returns {Object} A client object with methods to interact with DynamoDB. */ -const createClient = ( +const createClient = async ( log = console, dbClient = AWSXray.captureAWSv3Client(new DynamoDB()), docClient = DynamoDBDocument.from(dbClient, { @@ -37,12 +37,30 @@ const createClient = ( removeUndefinedValues: true, }, }), -) => ({ - scan: (params) => scan(docClient, params, log), - query: (params) => query(docClient, params, log), - getItem: (tableName, key) => getItem(docClient, tableName, key, log), - putItem: (tableName, item) => putItem(docClient, tableName, item, log), - removeItem: (tableName, key) => removeItem(docClient, tableName, key, log), -}); +) => { + // Add middleware to log connection details + dbClient.middlewareStack.add( + (next) => async (args) => { + // Resolve the endpoint dynamically + const region = await dbClient.config.region(); + console.log('DynamoDB Connection Details:'); + console.log('Region:', region); + + // Proceed to the next middleware + return next(args); + }, + { + step: 'initialize', // Log during the initialization phase + }, + ); + + return { + scan: (params) => scan(docClient, params, log), + query: (params) => query(docClient, params, log), + getItem: (tableName, key) => getItem(docClient, tableName, key, log), + putItem: (tableName, item) => putItem(docClient, tableName, item, log), + removeItem: (tableName, key) => removeItem(docClient, tableName, key, log), + }; +}; export { createClient };