Merge pull request #185 from MikroElektronika/topic/switch-to-dbp

Updated according to new DBP.
MikroElektronika · Dec 24, 2024 · a0f4b61 · a0f4b61
2 parents 756dc06 + 521b378
commit a0f4b61
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 32 deletions.
diff --git a/.github/workflows/checkIndexes.yaml b/.github/workflows/checkIndexes.yaml
@@ -56,7 +56,7 @@ jobs:
           FIX_ACTION=${{ github.event.inputs.fix }}  # Capture the fix input
           LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false)  # Negate the fix input
           echo "LOG_ONLY is set to $LOG_ONLY"
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
         continue-on-error: true # Ensure the workflow continues
 
       - name: Check Indexed Links - Test
@@ -65,7 +65,7 @@ jobs:
           FIX_ACTION=${{ github.event.inputs.fix }}  # Capture the fix input
           LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false)  # Negate the fix input
           echo "LOG_ONLY is set to $LOG_ONLY"
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
         continue-on-error: true # Ensure the workflow continues
 
   push_to_main_run:
@@ -90,12 +90,12 @@ jobs:
 
       - name: Check Indexed Links - Live
         run: |
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX"
         continue-on-error: true # Ensure the workflow continues
 
       - name: Check Indexed Links - Test
         run: |
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX"
         continue-on-error: true # Ensure the workflow continues
 
   scheduled_run:
@@ -120,10 +120,10 @@ jobs:
 
       - name: Check Indexed Links - Live
         run: |
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX"
         continue-on-error: true # Ensure the workflow continues
 
       - name: Check Indexed Links - Test
         run: |
-          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX"
+          python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX"
         continue-on-error: true # Ensure the workflow continues
diff --git a/scripts/build_message_web.py b/scripts/build_message_web.py
@@ -81,18 +81,19 @@ def fetch_current_indexed_click_boards(es : Elasticsearch, index_name):
 
     # Elasticsearch instance used for getting indexing info
     num_of_retries = 1
+    print("Trying to connect to ES.")
     while True:
-        print(f"Trying to connect to ES. Connection retry:  {num_of_retries}")
         es = Elasticsearch([os.environ['ES_HOST']], http_auth=(os.environ['ES_USER'], os.environ['ES_PASSWORD']))
         if es.ping():
             break
-        # Wait for 30 seconds and try again if connection fails
+        # Wait 1 second and try again if connection fails
         if 10 == num_of_retries:
             # Exit if it fails 10 times, something is wrong with the server
             raise ValueError("Connection to ES failed!")
+        print(f"Connection retry: {num_of_retries}")
         num_of_retries += 1
 
-        time.sleep(30)
+        time.sleep(1)
 
     current_date = datetime.now().strftime("%Y-%m-%d")
 

diff --git a/scripts/check_indexes.py b/scripts/check_indexes.py
@@ -23,9 +23,6 @@ def str2bool(v):
     parser.add_argument("es_host", help="ES instance host value", type=str)
     parser.add_argument("es_user", help="ES instance user value", type=str)
     parser.add_argument("es_password", help="ES instance password value", type=str)
-    parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str)
-    parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str)
-    parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str)
     parser.add_argument("es_index", help="ES instance index value", type=str)
     parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+")
     parser.add_argument("--log_only", help="If True, will not fix broken links, just log them to std out", type=str2bool, default=False)
@@ -37,11 +34,6 @@ def str2bool(v):
         index=args.es_index, token=args.gh_token
     )
 
-    me_es_instance = es.index(
-        es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password,
-        index=args.es_index, token=args.gh_token
-    )
-
     gh_instance = gh.repo(args.gh_repo, args.gh_token)
 
     es_instance.fetch(regex=args.es_regex)
@@ -63,7 +55,7 @@ def str2bool(v):
                     indexed_item['source']['download_link'] = url
                     if 'extra_information' not in indexed_item['source']: # Add extra information for Card or Board
                         extra_info.add(indexed_item['source'], args.gh_token, args.es_index)
-                    es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source'])
+                    es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
                 else:
                     print("%sWARNING: Asset \"%s\" has no \"gh_package_name\" in the index." % (es_instance.Colors.WARNING, indexed_item['source']['name']))
         else: ## code 200 - success, no need to reindex
@@ -73,24 +65,20 @@ def str2bool(v):
                     indexed_item['source'].update({"gh_package_name": package_name})
                     if 'extra_information' not in indexed_item['source']: # Add extra information for Card or Board
                         extra_info.add(indexed_item['source'], args.gh_token, args.es_index)
-                    es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source'])
+                    es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
                     print("%sINFO: Added \"gh_package_name\" to %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
                 else:
                     if package_name != indexed_item['source']['gh_package_name']:
                         indexed_item['source']['gh_package_name'] = package_name
                         if 'extra_information' not in indexed_item['source']: # Add extra information for Card or Board
                             extra_info.add(indexed_item['source'], args.gh_token, args.es_index)
-                        es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source'])
+                        es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
                         print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
                     else:
                         if 'extra_information' not in indexed_item['source']: # Add extra information for Card or Board
                             extra_info.add(indexed_item['source'], args.gh_token, args.es_index)
-                            es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source'])
+                            es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
             print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link']))
 
-        # For new elasticsearch DBP it is crucial not to use doc_type for indexing
-        me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
-        print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
-
     if err and args.log_only:
         sys.exit(-1)
diff --git a/scripts/classes/class_es.py b/scripts/classes/class_es.py
@@ -24,15 +24,16 @@ def init(es_host, es_user, es_password, retry=None):
         if retry:
             retry_check = retry
         num_of_retries = 1
+        print("Trying to connect to ES.")
         while True:
-            print(f"Trying to connect to ES. Connection retry:  {num_of_retries}")
             es = Elasticsearch([es_host], http_auth=(es_user, es_password))
             if es.ping():
                 break
-            # Wait for 1 second and try again if connection fails
+            # Wait 1 second and try again if connection fails
             if retry_check == num_of_retries:
                 # Exit if it fails 10 times, something is wrong with the server
                 raise ValueError("Connection to ES failed!")
+            print(f"Connection retry: {num_of_retries}")
             num_of_retries += 1
             es = None
 

diff --git a/scripts/index.py b/scripts/index.py
@@ -437,12 +437,12 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai
 
             # Index the document
             if doc:
-                resp = es.index(index=index_name, doc_type='necto_package', id=package_id, body=doc)
+                resp = es.index(index=index_name, doc_type=None, id=package_id, body=doc)
                 ## Special case for images, update live index elasticsearch base as well
                 ## Called only from board release workflow
                 if ('ES_INDEX_TEST' in os.environ) and ('ES_INDEX_LIVE' in os.environ):
                     if ('images' == name_without_extension) and (index_name == os.environ['ES_INDEX_TEST']):
-                        resp = es.index(index=os.environ['ES_INDEX_LIVE'], doc_type='necto_package', id=package_id, body=doc)
+                        resp = es.index(index=os.environ['ES_INDEX_LIVE'], doc_type=None, id=package_id, body=doc)
                 if doc['package_changed']:
                     logger.info(f"{resp["result"]} {resp['_id']}")
                     logger.info(f"Download link is {doc['download_link']}")
@@ -540,18 +540,19 @@ def str2bool(v):
 
     # Elasticsearch instance used for indexing
     num_of_retries = 1
+    print("Trying to connect to ES.")
     while True:
-        print(f"Trying to connect to ES. Connection retry:  {num_of_retries}")
         es = Elasticsearch([os.environ['ES_HOST']], http_auth=(os.environ['ES_USER'], os.environ['ES_PASSWORD']))
         if es.ping():
             break
-        # Wait for 30 seconds and try again if connection fails
+        # Wait 1 second and try again if connection fails
         if 10 == num_of_retries:
             # Exit if it fails 10 times, something is wrong with the server
             raise ValueError("Connection to ES failed!")
+        print(f"Connection retry: {num_of_retries}")
         num_of_retries += 1
 
-        time.sleep(30)
+        time.sleep(1)
 
     # Now index the new release
     index_release_to_elasticsearch(