From 4c7f5e12a041339b46710b73e3b95caa2b7ec359 Mon Sep 17 00:00:00 2001 From: "ivan.ruzavin" Date: Thu, 14 Nov 2024 13:02:53 +0100 Subject: [PATCH 1/2] Added workflow for copying indexes to DBP --- .github/workflows/copyIndexesToDBP.yaml | 94 +++++++++++++++++++++++++ scripts/classes/class_es.py | 19 +++-- scripts/index_to_me.py | 63 +++++++++++++++++ 3 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/copyIndexesToDBP.yaml create mode 100644 scripts/index_to_me.py diff --git a/.github/workflows/copyIndexesToDBP.yaml b/.github/workflows/copyIndexesToDBP.yaml new file mode 100644 index 000000000..b0d3ff7b7 --- /dev/null +++ b/.github/workflows/copyIndexesToDBP.yaml @@ -0,0 +1,94 @@ +name: Copy Indexes from AWS to DBP + +on: + workflow_dispatch: + inputs: + select_index: + type: choice + description: Copy Test and/or Live ES indexed items + options: + - Test + - Live + - Both + regex: + type: string + description: Regex to use when searching for indexed items + default: "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + +jobs: + manual_run: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r scripts/requirements/check_index.txt + + - name: Copy Indexed Links - Live + if: ${{ github.event.inputs.select_index == 'Live' || github.event.inputs.select_index == 'Both' }} + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" + + - name: Copy Indexed Links - Test + if: ${{ github.event.inputs.select_index == 'Test' || github.event.inputs.select_index == 'Both' }} + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" + + push_to_main_run: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r scripts/requirements/check_index.txt + + - name: Check Indexed Links - Live + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + + - name: Check Indexed Links - Test + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + + scheduled_run: + if: ${{ github.event_name == 'schedule' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r scripts/requirements/check_index.txt + + - name: Check Indexed Links - Live + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + + - name: Check Indexed Links - Test + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" diff --git a/scripts/classes/class_es.py b/scripts/classes/class_es.py index 149e4a1d6..731edae10 100644 --- a/scripts/classes/class_es.py +++ b/scripts/classes/class_es.py @@ -119,12 +119,21 @@ def create(self, doc_type, doc_id, doc_body): def update(self, doc_type, doc_id, doc_body): response = self.api_index(self.es_instance, self.index, doc_type, doc_id, doc_body) - if response['created'] and 'created' == response['result']: - print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) - elif not 'updated' == response['result']: - raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + if doc_type: + if response['created'] and 'created' == response['result']: + print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) + elif not 'updated' == response['result']: + raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + else: + print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) + # For new DBP elasticsearch there is no 'created' field in response, so we just check 'result' else: - print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) + if 'created' == response['result']: + print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) + elif not 'updated' == response['result']: + raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + else: + print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) def delete(self, doc_type, doc_id): response = self.es_instance.delete( diff --git a/scripts/index_to_me.py b/scripts/index_to_me.py new file mode 100644 index 000000000..06cd3625f --- /dev/null +++ b/scripts/index_to_me.py @@ -0,0 +1,63 @@ +import sys, json, argparse, requests + +import classes.class_gh as gh +import classes.class_es as es + +if __name__ == "__main__": + # Get arguments + parser = argparse.ArgumentParser(description="Upload directories as release assets.") + parser.add_argument("gh_repo", help="Github repository name, e.g., 'username/repo'", type=str) + parser.add_argument("gh_token", help="GitHub Token", type=str) + parser.add_argument("es_host", help="ES instance host value", type=str) + parser.add_argument("es_user", help="ES instance user value", type=str) + parser.add_argument("es_password", help="ES instance password value", type=str) + parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str) + parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str) + parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str) + parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+") + args = parser.parse_args() + + es_instance = es.index( + es_host=args.es_host, es_user=args.es_user, es_password=args.es_password, + index=args.es_index, token=args.gh_token + ) + + me_es_instance = es.index( + es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password, + index=args.es_index, token=args.gh_token + ) + + gh_instance = gh.repo(args.gh_repo, args.gh_token) + + es_instance.fetch(regex=args.es_regex) + + headers = { + 'Authorization': f'token {args.gh_token}' + } + + for indexed_item in es_instance.indexed_items: + asset_status = requests.get(indexed_item['source']['download_link'], headers=headers) + if es_instance.Status.ERROR.value == asset_status.status_code: ## code 404 - error, reindex with correct download link + print("%sERROR: Asset \"%s\" download link is incorrect. - %s" % (es_instance.Colors.FAIL, indexed_item['source']['name'], indexed_item['source']['download_link'])) + if 'gh_package_name' in indexed_item['source']: + url = gh_instance.asset_fetch_url_api(indexed_item['source']['gh_package_name'], loose=False) + indexed_item['source']['download_link'] = url + es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) + else: + print("%sWARNING: Asset \"%s\" has no \"gh_package_name\" in the index." % (es_instance.Colors.WARNING, indexed_item['source']['name'])) + else: ## code 200 - success, no need to reindex + package_name = (json.loads(asset_status.text))['name'] + if 'gh_package_name' not in indexed_item['source']: + indexed_item['source'].update({"gh_package_name": package_name}) + es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Added \"gh_package_name\" to %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) + else: + if package_name != indexed_item['source']['gh_package_name']: + indexed_item['source']['gh_package_name'] = package_name + es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) + print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link'])) + + # For new elasticsearch DBP it is crucial not to use doc_type for indexing + me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'], )) From 14aa790d2d2079978726d305866dfd2dd77414f2 Mon Sep 17 00:00:00 2001 From: "ivan.ruzavin" Date: Fri, 15 Nov 2024 09:11:47 +0100 Subject: [PATCH 2/2] Updated check index workflow with dbp indexing --- .github/workflows/checkIndexes.yaml | 15 ++-- .github/workflows/copyIndexesToDBP.yaml | 94 ------------------------- scripts/check_indexes.py | 12 ++++ scripts/index_to_me.py | 63 ----------------- 4 files changed, 21 insertions(+), 163 deletions(-) delete mode 100644 .github/workflows/copyIndexesToDBP.yaml delete mode 100644 scripts/index_to_me.py diff --git a/.github/workflows/checkIndexes.yaml b/.github/workflows/checkIndexes.yaml index d3d4df23f..7d361554b 100644 --- a/.github/workflows/checkIndexes.yaml +++ b/.github/workflows/checkIndexes.yaml @@ -26,6 +26,9 @@ on: schedule: - cron: "0/30 7-16 * * 1-5" # Every 30 minutes, between 07:00 AM and 04:59 PM, Monday through Friday +env: + GLOBAL_REGEX: "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + jobs: manual_run: if: ${{ github.event_name == 'workflow_dispatch' }} @@ -51,7 +54,7 @@ jobs: FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input echo "LOG_ONLY is set to $LOG_ONLY" - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test @@ -60,7 +63,7 @@ jobs: FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input echo "LOG_ONLY is set to $LOG_ONLY" - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" continue-on-error: true # Ensure the workflow continues push_to_main_run: @@ -83,12 +86,12 @@ jobs: - name: Check Indexed Links - Live run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues scheduled_run: @@ -111,10 +114,10 @@ jobs: - name: Check Indexed Links - Live run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues diff --git a/.github/workflows/copyIndexesToDBP.yaml b/.github/workflows/copyIndexesToDBP.yaml deleted file mode 100644 index b0d3ff7b7..000000000 --- a/.github/workflows/copyIndexesToDBP.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: Copy Indexes from AWS to DBP - -on: - workflow_dispatch: - inputs: - select_index: - type: choice - description: Copy Test and/or Live ES indexed items - options: - - Test - - Live - - Both - regex: - type: string - description: Regex to use when searching for indexed items - default: "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" - -jobs: - manual_run: - if: ${{ github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/check_index.txt - - - name: Copy Indexed Links - Live - if: ${{ github.event.inputs.select_index == 'Live' || github.event.inputs.select_index == 'Both' }} - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" - - - name: Copy Indexed Links - Test - if: ${{ github.event.inputs.select_index == 'Test' || github.event.inputs.select_index == 'Both' }} - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" - - push_to_main_run: - if: ${{ github.event_name == 'push' }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/check_index.txt - - - name: Check Indexed Links - Live - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" - - - name: Check Indexed Links - Test - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" - - scheduled_run: - if: ${{ github.event_name == 'schedule' }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/check_index.txt - - - name: Check Indexed Links - Live - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" - - - name: Check Indexed Links - Test - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board" diff --git a/scripts/check_indexes.py b/scripts/check_indexes.py index 4a98ee685..dda97b6cb 100644 --- a/scripts/check_indexes.py +++ b/scripts/check_indexes.py @@ -22,6 +22,9 @@ def str2bool(v): parser.add_argument("es_host", help="ES instance host value", type=str) parser.add_argument("es_user", help="ES instance user value", type=str) parser.add_argument("es_password", help="ES instance password value", type=str) + parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str) + parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str) + parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str) parser.add_argument("es_index", help="ES instance index value", type=str) parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+") parser.add_argument("--log_only", help="If True, will not fix broken links, just log them to std out", type=str2bool, default=False) @@ -33,6 +36,11 @@ def str2bool(v): index=args.es_index, token=args.gh_token ) + me_es_instance = es.index( + es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password, + index=args.es_index, token=args.gh_token + ) + gh_instance = gh.repo(args.gh_repo, args.gh_token) es_instance.fetch(regex=args.es_regex) @@ -68,5 +76,9 @@ def str2bool(v): print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link'])) + # For new elasticsearch DBP it is crucial not to use doc_type for indexing + me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) + if err and args.log_only: sys.exit(-1) diff --git a/scripts/index_to_me.py b/scripts/index_to_me.py deleted file mode 100644 index 06cd3625f..000000000 --- a/scripts/index_to_me.py +++ /dev/null @@ -1,63 +0,0 @@ -import sys, json, argparse, requests - -import classes.class_gh as gh -import classes.class_es as es - -if __name__ == "__main__": - # Get arguments - parser = argparse.ArgumentParser(description="Upload directories as release assets.") - parser.add_argument("gh_repo", help="Github repository name, e.g., 'username/repo'", type=str) - parser.add_argument("gh_token", help="GitHub Token", type=str) - parser.add_argument("es_host", help="ES instance host value", type=str) - parser.add_argument("es_user", help="ES instance user value", type=str) - parser.add_argument("es_password", help="ES instance password value", type=str) - parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str) - parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str) - parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str) - parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+") - args = parser.parse_args() - - es_instance = es.index( - es_host=args.es_host, es_user=args.es_user, es_password=args.es_password, - index=args.es_index, token=args.gh_token - ) - - me_es_instance = es.index( - es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password, - index=args.es_index, token=args.gh_token - ) - - gh_instance = gh.repo(args.gh_repo, args.gh_token) - - es_instance.fetch(regex=args.es_regex) - - headers = { - 'Authorization': f'token {args.gh_token}' - } - - for indexed_item in es_instance.indexed_items: - asset_status = requests.get(indexed_item['source']['download_link'], headers=headers) - if es_instance.Status.ERROR.value == asset_status.status_code: ## code 404 - error, reindex with correct download link - print("%sERROR: Asset \"%s\" download link is incorrect. - %s" % (es_instance.Colors.FAIL, indexed_item['source']['name'], indexed_item['source']['download_link'])) - if 'gh_package_name' in indexed_item['source']: - url = gh_instance.asset_fetch_url_api(indexed_item['source']['gh_package_name'], loose=False) - indexed_item['source']['download_link'] = url - es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) - else: - print("%sWARNING: Asset \"%s\" has no \"gh_package_name\" in the index." % (es_instance.Colors.WARNING, indexed_item['source']['name'])) - else: ## code 200 - success, no need to reindex - package_name = (json.loads(asset_status.text))['name'] - if 'gh_package_name' not in indexed_item['source']: - indexed_item['source'].update({"gh_package_name": package_name}) - es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) - print("%sINFO: Added \"gh_package_name\" to %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) - else: - if package_name != indexed_item['source']['gh_package_name']: - indexed_item['source']['gh_package_name'] = package_name - es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) - print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) - print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link'])) - - # For new elasticsearch DBP it is crucial not to use doc_type for indexing - me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) - print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'], ))