Updated check index workflow with dbp indexing

MikroElektronika · Nov 15, 2024 · 14aa790 · 14aa790
1 parent 4c7f5e1
commit 14aa790
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 163 deletions.
diff --git a/.github/workflows/checkIndexes.yaml b/.github/workflows/checkIndexes.yaml
@@ -26,6 +26,9 @@ on:
   schedule:
     - cron: "0/30 7-16 * * 1-5"  # Every 30 minutes, between 07:00 AM and 04:59 PM, Monday through Friday
 
+env:
+  GLOBAL_REGEX: "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
+
 jobs:
   manual_run:
     if: ${{ github.event_name == 'workflow_dispatch' }}
@@ -51,7 +54,7 @@ jobs:
           FIX_ACTION=${{ github.event.inputs.fix }}  # Capture the fix input
           LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false)  # Negate the fix input
           echo "LOG_ONLY is set to $LOG_ONLY"
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
         continue-on-error: true # Ensure the workflow continues
 
       - name: Check Indexed Links - Test
@@ -60,7 +63,7 @@ jobs:
           FIX_ACTION=${{ github.event.inputs.fix }}  # Capture the fix input
           LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false)  # Negate the fix input
           echo "LOG_ONLY is set to $LOG_ONLY"
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
         continue-on-error: true # Ensure the workflow continues
 
   push_to_main_run:
@@ -83,12 +86,12 @@ jobs:
 
       - name: Check Indexed Links - Live
         run: |
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX"
         continue-on-error: true # Ensure the workflow continues
 
       - name: Check Indexed Links - Test
         run: |
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX"
         continue-on-error: true # Ensure the workflow continues
 
   scheduled_run:
@@ -111,10 +114,10 @@ jobs:
 
       - name: Check Indexed Links - Live
         run: |
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX"
         continue-on-error: true # Ensure the workflow continues
 
       - name: Check Indexed Links - Test
         run: |
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX"
         continue-on-error: true # Ensure the workflow continues
diff --git a/.github/workflows/copyIndexesToDBP.yaml b/.github/workflows/copyIndexesToDBP.yaml
diff --git a/scripts/check_indexes.py b/scripts/check_indexes.py
@@ -22,6 +22,9 @@ def str2bool(v):
     parser.add_argument("es_host", help="ES instance host value", type=str)
     parser.add_argument("es_user", help="ES instance user value", type=str)
     parser.add_argument("es_password", help="ES instance password value", type=str)
+    parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str)
+    parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str)
+    parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str)
     parser.add_argument("es_index", help="ES instance index value", type=str)
     parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+")
     parser.add_argument("--log_only", help="If True, will not fix broken links, just log them to std out", type=str2bool, default=False)
@@ -33,6 +36,11 @@ def str2bool(v):
         index=args.es_index, token=args.gh_token
     )
 
+    me_es_instance = es.index(
+        es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password,
+        index=args.es_index, token=args.gh_token
+    )
+
     gh_instance = gh.repo(args.gh_repo, args.gh_token)
 
     es_instance.fetch(regex=args.es_regex)
@@ -68,5 +76,9 @@ def str2bool(v):
                         print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
             print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link']))
 
+        # For new elasticsearch DBP it is crucial not to use doc_type for indexing
+        me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
+        print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
+
     if err and args.log_only:
         sys.exit(-1)
diff --git a/scripts/index_to_me.py b/scripts/index_to_me.py