Merge pull request #11 from catenax-ng/main

Configuration update and API improvement

.github/workflows/BETA-xray-cucumber-integration.yaml

@@ -0,0 +1,52 @@
+name: IRS BETA Cucumber Integration test Xray execution
+
+on:
+ workflow_dispatch: # Trigger manually
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+
+ - name: Cache maven packages
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2
+
+ - name: Download Feature Files
+ id: download
+ env:
+ JIRA_USERNAME: ${{ secrets.ORG_IRS_JIRA_USERNAME }}
+ JIRA_PASSWORD: ${{ secrets.ORG_IRS_JIRA_PASSWORD }}
+ # JIRA filter 11590: project = "TRI" AND type = Test AND "Test Type" = Cucumber AND environment ~ Beta
+ # Downloads all feature files of cucumber tests inside TRI project
+ run: |
+ export HTTP_RESULT=$(curl -s --show-error -w "%{http_code}" -u $JIRA_USERNAME:$JIRA_PASSWORD "https://jira.catena-x.net/rest/raven/1.0/export/test?filter=11590&fz=true" -o features.zip)
+ [[ $HTTP_RESULT == 200 || $HTTP_RESULT == 400 ]]
+ echo "::set-output name=http_response::$HTTP_RESULT"
+
+ - name: Build with Maven
+ if: ${{ steps.download.outputs.http_response == '200' }}
+ env:
+ KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_OAUTH2_CLIENT_SECRET_BETA }}
+ run: |
+ unzip -o features.zip -d cucumber-tests/src/test/resources/features
+ mvn --batch-mode clean install -pl cucumber-tests,irs-models -D"cucumber.filter.tags"="not @Ignore and @INTEGRATION_TEST"
+
+ - name: Submit results to Xray
+ if: ${{ always() && steps.download.outputs.http_response == '200' }}
+ env:
+ JIRA_USERNAME: ${{ secrets.ORG_IRS_JIRA_USERNAME }}
+ JIRA_PASSWORD: ${{ secrets.ORG_IRS_JIRA_PASSWORD }}
+ run: |
+ curl --request POST -u $JIRA_USERNAME:$JIRA_PASSWORD --header 'Content-Type: application/json' --data-binary '@cucumber-tests/report.json' "https://jira.catena-x.net/rest/raven/1.0/import/execution/cucumber"

.github/workflows/irs-build.yml

@@ -84,6 +84,8 @@ jobs:
  VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
  # Strip "v" prefix from tag name
  [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+ # Support PR ref versions
+ [[ "${{ github.ref }}" == "refs/pull/"* ]] && VERSION=PR-$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\)/merge,\1,')
  # Use Docker `latest` tag convention
  [ "$VERSION" == "main" ] && VERSION=latest
  echo IMAGE_ID=$IMAGE_ID
@@ -115,6 +117,8 @@ jobs:
  VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
  # Strip "v" prefix from tag name
  [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+ # Support PR ref versions
+ [[ "${{ github.ref }}" == "refs/pull/"* ]] && VERSION=PR-$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\)/merge,\1,')
  # Use Docker `latest` tag convention
  [ "$VERSION" == "main" ] && VERSION=latest
  echo IMAGE_ID=$IMAGE_ID

.github/workflows/publish-documentation.yaml

@@ -24,6 +24,12 @@ jobs:
  - name: Setup Graphviz
  uses: ts-graphviz/setup-graphviz@v1
 
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+
  - name: Cache maven packages
  uses: actions/cache@v3
  with:
@@ -42,10 +48,24 @@ jobs:
  run: |
  mvn -f docs/pom.xml --batch-mode generate-resources
 
+ - name: Install pandoc
+ run: |
+ sudo apt-get install pandoc 
+ which pandoc
+
+ - name: Convert HTML to markdown
+ env:
+ FILES: "administration/administration-guide arc42/full api-specification/api-specification"
+ run: |
+ cd docs/target/generated-docs
+ for file in ${FILES}; do \
+ pandoc -f html -t markdown_strict --wrap=preserve "${file}.html" -o "${file}.md"; \
+ sed -i 's/](#_/](#/' "${file}.md"; \
+ sed -i -E '/\[.*\]\(#.*\)/ s/_/-/g' "${file}.md"; done
+
  - name: GitHub Pages action
  uses: peaceiris/actions-gh-pages@v3.9.0
  with:
  github_token: ${{ secrets.GITHUB_TOKEN }}
  publish_dir: "./docs/target/generated-docs"
  destination_dir: "docs"
-

.github/workflows/synch-env-branch.yml

@@ -1,6 +1,7 @@
 name: "Sync main to IRS-Env"
 
 on:
+ workflow_dispatch:
  push:
  branches:
  - main
@@ -20,7 +21,8 @@ jobs:
  git config user.email github-actions@github.com
  git fetch --no-tags --progress
  git checkout --progress -b catena-x-environments origin/catena-x-environments
- git merge origin/main
+ git fetch origin --recurse-submodules=no --progress --prune
+ git merge origin/main -v --allow-unrelated-histories
  git push
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/trivy.yml

@@ -66,6 +66,8 @@ jobs:
  VERSION=$(echo "${{ github.head_ref || github.ref }}" | sed -e 's,.*/\(.*\),\1,')
  # Strip "v" prefix from tag name
  [[ "${{ github.head_ref || github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+ # Support PR ref versions
+ [[ "${{ github.ref }}" == "refs/pull/"* ]] && VERSION=PR-$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\)/merge,\1,')
  # Use Docker `latest` tag convention
  [ "$VERSION" == "main" ] && VERSION=latest
  echo VERSION=$VERSION

CHANGELOG.md

@@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Change 'jobParameter' to 'parameter' in GET calls in IRS API
+- Change 'jobStates' to 'states' request parameter in GET call for jobs by states, 'jobStates' is now deprecated
+- REST clients for DTR, SemHub and BPDM now use their own RestTemplates and configuration
+- application.yaml received some documentation
 
 ### Known knowns
 - PLACEHOLDER REMOVE IF EMPTY: risks that were introduced or discovered in the release and are known but not resolved

api-tests/irs-api-tests.tavern.yaml

@@ -799,7 +799,7 @@ stages:
  save:
  json:
  job_id: id
- delay_after: 340
+ delay_after: 400
 
  - name: fetch response for created job and check details
  request:
@@ -914,7 +914,7 @@ stages:
 
  - name: get all COMPLETED jobs
  request:
- url: "{tavern.env_vars.IRS_HOST}/irs/jobs?states=COMPLETED"
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs?jobStates=COMPLETED"
  method: GET
  headers:
  content-type: application/json
@@ -959,7 +959,7 @@ stages:
 
  - name: get all ERROR jobs
  request:
- url: "{tavern.env_vars.IRS_HOST}/irs/jobs?states=ERROR"
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs?jobStates=ERROR"
  method: GET
  headers:
  content-type: application/json
@@ -1004,7 +1004,7 @@ stages:
 
  - name: get all INITIAL jobs
  request:
- url: "{tavern.env_vars.IRS_HOST}/irs/jobs?states=INITIAL"
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs?jobStates=INITIAL"
  method: GET
  headers:
  content-type: application/json
@@ -1069,7 +1069,7 @@ stages:
 
  - name: get all RUNNING jobs
  request:
- url: "{tavern.env_vars.IRS_HOST}/irs/jobs?states=RUNNING"
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs?jobStates=RUNNING"
  method: GET
  headers:
  content-type: application/json
@@ -1291,7 +1291,7 @@ stages:
 
  - name: get all jobs with status UNSAVED, INITIAL, RUNNING, TRANSFERS_FINISHED and CANCELED
  request:
- url: "{tavern.env_vars.IRS_HOST}/irs/jobs?states=UNSAVED, INITIAL, RUNNING, TRANSFERS_FINISHED, CANCELED"
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs?jobStates=UNSAVED, INITIAL, RUNNING, TRANSFERS_FINISHED, CANCELED"
  method: GET
  headers:
  content-type: application/json
@@ -1335,7 +1335,7 @@ stages:
 
  - name: get all jobs with status COMPLETED and ERROR
  request:
- url: "{tavern.env_vars.IRS_HOST}/irs/jobs?states=COMPLETED, ERROR"
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs?jobState=COMPLETED, ERROR"
  method: GET
  headers:
  content-type: application/json
@@ -1379,7 +1379,7 @@ stages:
 
  - name: get all jobs with status COMPLETED and ERROR
  request:
- url: "{tavern.env_vars.IRS_HOST}/irs/jobs?states=COMPLETED, ERROR"
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs?jobStates=COMPLETED, ERROR"
  method: GET
  headers:
  content-type: application/json
@@ -1513,5 +1513,91 @@ stages:
  - function: api-tests.tavern_helpers:relationships_are_empty
  - function: api-tests.tavern_helpers:check_timestamps_for_completed_jobs
  - function: api-tests.tavern_helpers:errors_for_unknown_globalAssetId_are_correct
+ headers:
+ content-type: application/json
+
+
+---
+
+
+test_name: Make sure pagination details exist as expected
+
+strict:
+ - headers:off
+ - json:off
+
+stages:
+ - name: authenticate and fetch access token
+ request:
+ url: "{tavern.env_vars.KEYCLOAK_HOST}"
+ method: POST
+ data:
+ grant_type: client_credentials
+ client_id: "{tavern.env_vars.KEYCLOAK_CLIENT_ID}"
+ client_secret: "{tavern.env_vars.KEYCLOAK_CLIENT_SECRET}"
+ response:
+ status_code: 200
+ json:
+ token_type: Bearer
+ headers:
+ content-type: application/json
+ save:
+ json:
+ access_token: access_token
+
+ - name: get all jobs
+ request:
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs"
+ method: GET
+ headers:
+ content-type: application/json
+ Authorization: "Bearer {access_token:s}"
+ response:
+ status_code: 200
+ verify_response_with:
+ function: api-tests.tavern_helpers:check_pagination_details_exists
+ headers:
+ content-type: application/json
+
+
+---
+
+
+test_name: Make sure pagination can be requested as expected
+
+strict:
+ - headers:off
+ - json:off
+
+stages:
+ - name: authenticate and fetch access token
+ request:
+ url: "{tavern.env_vars.KEYCLOAK_HOST}"
+ method: POST
+ data:
+ grant_type: client_credentials
+ client_id: "{tavern.env_vars.KEYCLOAK_CLIENT_ID}"
+ client_secret: "{tavern.env_vars.KEYCLOAK_CLIENT_SECRET}"
+ response:
+ status_code: 200
+ json:
+ token_type: Bearer
+ headers:
+ content-type: application/json
+ save:
+ json:
+ access_token: access_token
+
+ - name: get all jobs with requested pagination settings
+ request:
+ url: "{tavern.env_vars.IRS_HOST}/irs/jobs?page=1&size=3&sort=jobCompleted,asc"
+ method: GET
+ headers:
+ content-type: application/json
+ Authorization: "Bearer {access_token:s}"
+ response:
+ status_code: 200
+ verify_response_with:
+ function: api-tests.tavern_helpers:check_pagination_is_requested_correctly
  headers:
  content-type: application/json

api-tests/tavern_helpers.py

@@ -75,16 +75,18 @@ def errors_for_unknown_globalAssetId_are_correct(response):
 
 
 def status_of_jobs_are_as_expected(response, expected_status):
- print(response.json())
- for i in response.json():
+ job_list = response.json().get("content")
+ print("Job Liste: ", job_list)
+ for i in job_list:
  actual_status = i.get("state")
  print(f"Asserting expected status '{expected_status}' to be equal to actual status '{actual_status}'")
  assert expected_status in actual_status
 
 
 def status_of_all_jobs_are_given(response):
- print(response.json())
- for i in response.json():
+ job_list = response.json().get("content")
+ print("Job Liste: ", job_list)
+ for i in job_list:
  actual_status = i.get("state")
  assert any(
  ["COMPLETED" in actual_status, "ERROR" in actual_status, "INITIAL" in actual_status, "CANCELED" in actual_status, "RUNNING" in actual_status]
@@ -125,30 +127,50 @@ def check_timestamps_for_not_completed_jobs(response):
 
 
 def check_startedOn_timestamp_exists(response):
- for i in response.json():
+ response_list = response.json().get("content")
+ for i in response_list:
  startedOn = i.get("startedOn")
  print("Check if startedOn timestamp is existing.")
  assert startedOn is not None
 
 
 def check_completedOn_timestamp_not_exists(response):
- for i in response.json():
+ response_list = response.json().get("content")
+ for i in response_list:
  completedOn = i.get("completedOn")
  print("Check if completedOn timestamp is missing.")
  assert completedOn is None
 
 
 def check_completedOn_timestamp_exists(response):
- for i in response.json():
+ response_list = response.json().get("content")
+ for i in response_list:
  completedOn = i.get("completedOn")
  print("Check if completedOn timestamp is existing.")
  assert completedOn is not None
 
 
-
 def check_startedOn_is_smaller_than_completedOn(response):
- for i in response.json():
+ response_list = response.json().get("content")
+ for i in response_list:
  completedOn_timestamp = datetime.strptime(i.get("completedOn")[:26], '%Y-%m-%dT%H:%M:%S.%f').timestamp()
  startedOn_timestamp = datetime.strptime(i.get("startedOn")[:26], '%Y-%m-%dT%H:%M:%S.%f').timestamp()
  print(f"Check if startedOn timestamp '{startedOn_timestamp}' is smaller than completedOn timestamp '{completedOn_timestamp}'")
- assert startedOn_timestamp < completedOn_timestamp
+ assert startedOn_timestamp < completedOn_timestamp
+
+
+def check_pagination_details_exists(response):
+ response_list = response.json()
+ assert response_list.get("pageNumber") == 0
+ assert response_list.get("pageCount") > 0
+ assert response_list.get("pageSize") == 20
+ assert response_list.get("totalElements") > 0
+
+
+def check_pagination_is_requested_correctly(response):
+ response_list = response.json()
+ assert response_list.get("pageNumber") == 1
+ assert response_list.get("pageCount") > 0
+ assert response_list.get("pageSize") == 3
+ assert response_list.get("totalElements") > 0
+ assert len(response.json().get("content")) == 3