fix: work in progress, add new notebook

NASA-IMPACT · May 21, 2024 · dfaff7f · dfaff7f
1 parent 22f374a
commit dfaff7f
Show file tree

Hide file tree

Showing 2 changed files with 505 additions and 8 deletions.
diff --git a/transformation-scripts/collection-and-item-workflows-ingest.ipynb b/transformation-scripts/collection-and-item-workflows-ingest.ipynb
@@ -67,7 +67,7 @@
     "collection_json_file_paths = glob.glob(\"../ingestion-data/collections/*.json\")\n",
     "filtered_collection_file_paths_list = [\n",
     "    item\n",
-    "    for item in json_file_paths\n",
+    "    for item in collection_json_file_paths\n",
     "    if all(\n",
     "        excluded_collections not in item\n",
     "        for excluded_collections in excluded_collections\n",
@@ -169,14 +169,8 @@
    "outputs": [],
    "source": [
     "test_endpoint = \"https://test.openveda.cloud\"\n",
-    "test_client_id = \"CHANGE ME\"\n",
-    "test_user_pool_id = \"CHANGE ME\"\n",
-    "test_identity_pool_id = \"CHANGE ME\"\n",
     "\n",
     "mcp_prod_endpoint = \"https://openveda.cloud\"\n",
-    "mcp_prod_client_id = \"CHANGE ME\"\n",
-    "mcp_prod_user_pool_id = \"CHANGE ME\"\n",
-    "mcp_prod_identity_pool_id = \"CHANGE ME\"\n",
     "\n",
     "print(f\"TESTING MODE? {testing_mode}\")\n",
     "if testing_mode:\n",
@@ -307,7 +301,8 @@
    "outputs": [],
    "source": [
     "test_file_paths_and_collection_ids = [file_paths_and_collection_ids[0]]\n",
-    "test_discovery_item = [f\"../ingestion-data/production/discovery-items/{file_paths_and_collection_ids[0].get(\"collectionId\")}.json\"]\n",
+    "test_discovery_json_path = f\"../ingestion-data/production/discovery-items/{file_paths_and_collection_ids[0].get(\"collectionId\")}.json\"\n",
+    "test_discovery_item = [test_discovery_json_path]\n",
     "\n",
     "print(test_discovery_item)\n",
     "print(test_file_paths_and_collection_ids)\n",