fix: migration script for v2

adobe · Dec 10, 2024 · f8b9c20 · f8b9c20
1 parent dbbd150
commit f8b9c20
Showing 1 changed file with 243 additions and 3 deletions.
diff --git a/packages/spacecat-shared-data-access/migrate_v2.sh b/packages/spacecat-shared-data-access/migrate_v2.sh
@@ -10,11 +10,19 @@ SITE_TABLE="spacecat-services-sites-dev"
 ORGANIZATION_TABLE="spacecat-services-organizations-dev"
 CONFIGURATION_TABLE="spacecat-services-configurations-dev"
 DATA_TABLE="spacecat-services-data-dev"
+EXPERIMENT_TABLE="spacecat-services-experiments-dev"
+SITE_CANDIDATE_TABLE="spacecat-services-site-candidates-dev"
+TOP_PAGES_TABLE="spacecat-services-site-top-pages-dev"
+KEY_EVENTS_TABLE="spacecat-services-key-events-dev"
 
 # Fetch all sites
 SITES=$($AWS_CMD scan --table-name $SITE_TABLE)
 ORGANIZATIONS=$($AWS_CMD scan --table-name $ORGANIZATION_TABLE)
 CONFIGURATIONS=$($AWS_CMD scan --table-name $CONFIGURATION_TABLE)
+EXPERIMENTS=$($AWS_CMD scan --table-name $EXPERIMENT_TABLE)
+SITE_CANDIDATES=$($AWS_CMD scan --table-name $SITE_CANDIDATE_TABLE)
+TOP_PAGES=$($AWS_CMD scan --table-name $TOP_PAGES_TABLE)
+KEY_EVENTS=$($AWS_CMD scan --table-name $KEY_EVENTS_TABLE)
 
 # Migrate each site
 echo "$SITES" | jq -c '.Items[]' | while read -r site; do
@@ -68,6 +76,7 @@ EOF
 
     $AWS_LOCAL_CMD put-item --table-name $DATA_TABLE --item "$MIGRATED_SITE"
 done
+echo "Migrated sites successfully."
 
 # Migrate each organization
 echo "$ORGANIZATIONS" | jq -c '.Items[]' | while read -r org; do
@@ -87,7 +96,7 @@ echo "$ORGANIZATIONS" | jq -c '.Items[]' | while read -r org; do
     "organizationId": {"S": "$ORG_ID"},
     "imsOrgId": {"S": "$IMS_ORG_ID"},
     "name": {"S": "$NAME"},
-    "gsi1pk": {"S": "$GSI1PK"},
+    "gsi1pk": {"S": "$ORGANIZATION_GSI1PK"},
     "gsi1sk": {"S": "$ORGANIZATION_GSI1SK"},
     "pk": {"S": "$ORGANIZATION_PK"},
     "sk": {"S": "$ORGANIZATION_SK"},
@@ -104,9 +113,240 @@ echo "$ORGANIZATIONS" | jq -c '.Items[]' | while read -r org; do
 EOF
 )
 
-    # Insert migrated organization data into the organization table
+    # Insert migrated organization into the data table
     $AWS_LOCAL_CMD put-item --table-name $DATA_TABLE --item "$MIGRATED_ORG"
 
 done
-#TODO Migrate each configuration, top page, etc
+echo "Migrated organizations successfully."
+
+# Migrate configurations
+echo "$CONFIGURATIONS" | jq -c '.Items[]' | while read -r configuration; do
+    VERSION=$(echo $configuration | jq -r '.version.N')
+    CONFIGURATION_ID=$(uuidgen)
+    CONFIGURATION_PK="\$spacecat#configurationId_$CONFIGURATION_ID"
+    CONFIGURATION_SK="\$configuration_1"
+    CONFIGURATION_GSI1PK="all_configurations"
+    CONFIGURATION_GSI1SK="\$configuration_1#version_${VERSION}"
+    CREATED_AT=$(echo $configuration | jq -r '.createdAt.S')
+    UPDATED_AT=$(echo $configuration | jq -r '.updatedAt.S')
+    QUEUES=$(echo $configuration | jq -r '.queues // {"M": {}}')
+    JOBS=$(echo $configuration | jq -r '.jobs // {"L": {}}')
+    HANDLERS=$(echo $configuration | jq -r '.handlers // {"M": {}}')
+    SLACK_ROLES=$(echo $configuration | jq -r '.slackRoles // {"M": {}}')
+ MIGRATED_CONFIGURATION=$(cat <<EOF
+{
+    "configurationId": {"S": "$CONFIGURATION_ID"},
+    "gsi1pk": {"S": "$CONFIGURATION_GSI1PK"},
+    "pk": {"S": "$CONFIGURATION_PK"},
+    "sk": {"S": "$CONFIGURATION_SK"},
+    "__edb_e__": {"S": "Configuration"},
+    "__edb_v__": {"N": "1"},
+    "createdAt": {"S": "$CREATED_AT"},
+    "updatedAt": {"S": "$UPDATED_AT"},
+    "queues": $QUEUES,
+    "jobs": $JOBS,
+    "handlers": $HANDLERS,
+    "slackRoles": $SLACK_ROLES
+}
+
+EOF
+)
+
+    # Insert migrated organization into the data table
+    $AWS_LOCAL_CMD put-item --table-name $DATA_TABLE --item "$MIGRATED_CONFIGURATION"
+
+done
+echo "Migrated configurations successfully."
+
+# Migrate each experiment
+echo "$EXPERIMENTS" | jq -c '.Items[]' | while read -r experiment; do
+    EXPERIMENT_ID=$(echo $experiment | jq -r '.experimentId.S')
+    EXP_ID=$(uuidgen)
+    SITE_ID=$(echo $experiment | jq -r '.siteId.S')
+    EXPERIMENT_PK="\$spacecat#experimentId_$EXP_ID"
+    EXPERIMENT_SK="\$experiment_1"
+    EXPERIMENTS_GSI1PK="\$spacecat#siteid_$SITE_ID"
+    EXPERIMENT_GSI1SK="$EXPERIMENT_SK#exp_id_${EXP_ID}#url_${URL}#updatedAt_${UPDATED_AT}"
+    START_DATE=$(echo $experiment | jq -r '.startDate.S')
+    STATUS=$(echo $experiment | jq -r '.status.S')
+    TYPE=$(echo $experiment | jq -r '.type.S')
+    START_DATE=$(echo $experiment | jq -r '.startDate.S')
+    END_DATE=$(echo $experiment | jq -r '.endDate.S')
+    URL=$(echo $experiment | jq -r '.url.S')
+    VARIANTS=$(echo $experiment | jq -r '.variants // {"L": []}')
+    NAME=$(echo $experiment | jq -r '.name.S')
+    CONVERSION_EVENT_NAME=$(echo $experiment | jq -r '.conversionEventName // empty')
+    CONVERSION_EVENT_VALUE=$(echo $experiment | jq -r '.conversionEventValue // empty')
+    CREATED_AT=$(echo $experiment | jq -r '.createdAt.S')
+    UPDATED_AT=$(echo $experiment | jq -r '.updatedAt.S')
+    UPDATED_BY=$(echo $experiment | jq -r '.updatedBy.S')
+ MIGRATED_EXPERIMENT=$(cat <<EOF
+{
+    "experimentId": {"S": "$EXPERIMENT_ID"},
+    "gsi1pk": {"S": "$EXPERIMENT_GSI1PK"},
+    "gsi1sk": {"S": "$EXPERIMENT_GSI1SK"},
+    "expId": {"S": "$EXP_ID"},
+    "siteId": {"S": "$SITE_ID"},
+    "startDate": {"S": "$START_DATE"},
+    "status": {"S": "$STATUS"},
+    "type": {"S": "$TYPE"},
+    "endDate": {"S": "$END_DATE"},
+    "url": {"S": "$URL"},
+    "variants": $VARIANTS,
+    "name": {"S": "$NAME"},
+    "conversionEventName": {"S": "$CONVERSION_EVENT_NAME"},
+    "conversionEventValue": {"S": "$CONVERSION_EVENT_VALUE"},
+    "pk": {"S": "$EXPERIMENT_PK"},
+    "sk": {"S": "$EXPERIMENT_SK"},
+    "__edb_e__": {"S": "Experiment"},
+    "__edb_v__": {"N": "1"},
+    "createdAt": {"S": "$CREATED_AT"},
+    "updatedAt": {"S": "$UPDATED_AT"},
+    "config": $CONFIG
+}
+
+EOF
+)
+
+    # Insert migrated experiment into the data table
+    $AWS_LOCAL_CMD put-item --table-name $DATA_TABLE --item "$MIGRATED_EXPERIMENT"
+
+done
+echo "Migrated experiments successfully."
+
+# Migrate each site candidate
+echo "$SITE_CANDIDATES" | jq -c '.Items[]' | while read -r site_candidate; do
+    SITE_CANDIDATE_ID=$(uuidgen)
+    SITE_CANDIDATE_PK="\$spacecat#sitecandidateId_$SITE_CANDIDATE_ID"
+    SITE_CANDIDATE_SK="\$sitecandidate_1"
+    CREATED_AT=$(echo $site_candidate | jq -r '.createdAt.S')
+    UPDATED_AT=$(echo $site_candidate | jq -r '.updatedAt.S')
+    UPDATED_BY=$(echo $site_candidate | jq -r '.updatedBy.S')
+    SITE_CANDIDATE_GSI1PK="all_sitecandidates"
+    SITE_CANDIDATE_GSI1SK="\$sitecandidate_1#baseurl_${BASE_URL}"
+    SITE_CANDIDATE_GSI2PK="\$spacecat#siteid_$SITE_ID"
+    SITE_CANDIDATE_GSI2SK="\$sitecandidate_1#updatedat_${UPDATED_AT}"
+    SITE_ID=$(echo $site_candidate | jq -r '.siteId.S')
+    BASE_URL=$(echo $site_candidate | jq -r '.baseURL.S')
+    SOURCE=$(echo $site_candidate | jq -r '.source.S')
+    HLX_CONFIG=$(echo $site_candidate | jq -r '.hlxConfig // {"M": {}}')
+    STATUS=$(echo $site_candidate | jq -r '.status.S')
+ MIGRATED_SITE_CANDIDATE=$(cat <<EOF
+{
+    "siteCandidateId": {"S": "$SITE_CANDIDATE_ID"},
+    "gsi1pk": {"S": "$SITE_CANDIDATE_PK"},
+    "gsi1sk": {"S": "$SITE_CANDIDATE_SK"},
+    "gsi2pk": {"S": "$SITE_CANDIDATE_GSI2PK"},
+    "gsi2sk": {"S": "$SITE_CANDIDATE_GSI2SK"},
+    "siteId": {"S": "$SITE_ID"},
+    "baseURL": {"S": "$BASE_URL"},
+    "source": {"S": "$SOURCE"},
+    "status": {"S": "$STATUS"},
+    "hlxConfig": $HLX_CONFIG,
+    "pk": {"S": "$SITE_CANDIDATE_PK"},
+    "sk": {"S": "$SITE_CANDIDATE_SK"},
+    "__edb_e__": {"S": "SiteCandidate"},
+    "__edb_v__": {"N": "1"},
+    "createdAt": {"S": "$CREATED_AT"},
+    "updatedBy": {"S": "$UPDATED_BY"},
+    "updatedAt": {"S": "$UPDATED_AT"},
+    "config": $CONFIG
+}
+
+EOF
+)
+
+    # Insert migrated site candidate into the data table
+    $AWS_LOCAL_CMD put-item --table-name $DATA_TABLE --item "$MIGRATED_SITE_CANDIDATE"
+
+done
+echo "Migrated site candidates successfully."
+
+# Migrate each top page
+echo "$TOP_PAGES" | jq -c '.Items[]' | while read -r top_page; do
+    TOP_PAGE_ID=$(uuidgen)
+    SITE_ID=$(echo $top_page | jq -r '.siteId.S')
+    GEOGRAPHY=$(echo $top_page | jq -r '.geo.S')
+    IMPORTED_AT=$(echo $top_page | jq -r '.importedAt.S')
+    SOURCE=$(echo $top_page | jq -r '.source.S')
+    TOP_KEYWORD=$(echo $top_page | jq -r '.topKeyword.S')
+    TRAFFIC=$(echo $top_page | jq -r '.traffic.N')
+    URL=$(echo $top_page | jq -r '.url.S')
+    TOP_PAGE_PK="\$spacecat#sitetoppageId_$TOP_PAGE_ID"
+    TOP_PAGE_SK="\$sitetoppage_1"
+    TOP_PAGE_GSI1PK="\$spacecat#siteid_$SITE_ID"
+    TOP_PAGE_GSI1SK="$TOP_PAGE_SK#source_${SOURCE}#geo_${GEOGRAPHY}#traffic_${TRAFFIC}"
+    CREATED_AT=$(echo $top_page | jq -r '.createdAt.S')
+    UPDATED_AT=$(echo $top_page | jq -r '.updatedAt.S')
+    CONFIG=$(echo $top_page | jq -r '.config // {"M": {}}')
+ MIGRATED_TOP_PAGE=$(cat <<EOF
+{
+    "siteTopPageId": {"S": "$TOP_PAGE_ID"},
+    "siteId": {"S": "$SITE_ID"},
+    "geo": {"S": "$GEOGRAPHY"},
+    "importedAt": {"S": "$IMPORTED_AT"},
+    "source": {"S": "$SOURCE"},
+    "topKeyword": {"S": "$TOP_KEYWORD"},
+    "traffic": {"N": "$TRAFFIC"},
+    "url": {"S": "$URL"},
+    "gsi1pk": {"S": "$TOP_PAGE_PK"},
+    "gsi1sk": {"S": "$TOP_PAGE_SK"},
+    "pk": {"S": "$TOP_PAGE_PK"},
+    "sk": {"S": "$TOP_PAGE_SK"},
+    "__edb_e__": {"S": "SiteTopPage"},
+    "__edb_v__": {"N": "1"},
+    "createdAt": {"S": "$CREATED_AT"},
+    "updatedAt": {"S": "$UPDATED_AT"},
+}
+
+EOF
+)
+
+    # Insert migrated top page into the data table
+    $AWS_LOCAL_CMD put-item --table-name $DATA_TABLE --item "$MIGRATED_TOP_PAGE"
+
+done
+echo "Migrated top pages successfully."
+
+# Migrate each key event
+echo "$KEY_EVENTS" | jq -c '.Items[]' | while read -r key_event; do
+    KEY_EVENT_ID=$(uuidgen)
+    SITE_ID=$(echo $key_event | jq -r '.siteId.S')
+    KEY_EVENT_PK="\$spacecat#keyeventid_$KEY_EVENT_ID"
+    KEY_EVENT_SK="\$keyevent_1"
+    CREATED_AT=$(echo $key_event | jq -r '.createdAt.S')
+    UPDATED_AT=$(echo $key_event | jq -r '.updatedAt.S')
+    NAME=$(echo $key_event | jq -r '.name.S')
+    TYPE=$(echo $key_event | jq -r '.type.S')
+    TIME=$(echo $key_event | jq -r '.time.S')
+    KEY_EVENT_GSI1PK="\$spacecat#siteid_$SITE_ID"
+    KEY_EVENT_GSI1SK="$KEY_EVENT_SK#time_${TIME}"
+    MIGRATED_KEY_EVENT=$(cat <<EOF
+{
+    "keyEventId": {"S": "$KEY_EVENT_ID"},
+    "gsi1pk": {"S": "$KEY_EVENT_PK"},
+    "gsi1sk": {"S": "$KEY_EVENT_SK"},
+    "siteId": {"S": "$SITE_ID"},
+    "pk": {"S": "$KEY_EVENT_PK"},
+    "sk": {"S": "$KEY_EVENT_SK"},
+    "__edb_e__": {"S": "KeyEvent"},
+    "__edb_v__": {"N": "1"},
+    "name": {"S": "$NAME"},
+    "type": {"S": "$TYPE"},
+    "time": {"S": "$TIME"},
+    "createdAt": {"S": "$CREATED_AT"},
+    "updatedAt": {"S": "$UPDATED_AT"},
+    "config": $CONFIG
+}
+
+EOF
+)
+
+    # Insert migrated key event into the data table
+    $AWS_LOCAL_CMD put-item --table-name $DATA_TABLE --item "$MIGRATED_KEY_EVENT"
+
+done
+    })
+
+
 echo "Migration completed successfully."