Merge pull request #18782 from anjuchamantha/master

[FAPI Conformance] Add email and Gchat sending option with test result summary

.github/workflows/fapi-oidc-conformance-test.yml

@@ -1,6 +1,6 @@
 # This workflow will test IS for OIDC FAPI conformance
 
-name: FAPI-OIDC-Conformance-Test
+name: FAPI OIDC Conformance Test
 
 on:
  schedule:
@@ -18,6 +18,14 @@ on:
  conformance-suite-version:
  description: 'Conformance suite branch to clone in https://gitlab.com/openid/conformance-suite.git (Ex: release-v5.1.10). If not provided, latest release tag branch is used.'
  required: false
+ send-email:
+ description: 'Send test results to email'
+ required: true
+ default: 'no'
+ send-chat:
+ description: 'Send test results to google chat'
+ required: true
+ default: 'yes'
 
 jobs:
  build:
@@ -216,7 +224,70 @@ jobs:
  with:
  name: test-logs
  path: ./*log.txt
-
-
+
+ - name: Send Email
+ if: always()
+ run: |
+ INPUT=${{github.event.inputs.send-email}}
+ if [[ -z "${INPUT}" ]]; then
+ INPUT="no"
+ fi
+ SEND_EMAIL=${INPUT^^}
+ if [ $SEND_EMAIL == "YES" ]; then
+ echo "============="
+ echo "Sending Email"
+ echo "============="
+ CONFORMANCE_SUITE_URL=https://localhost:8443
+ INPUT_TAG=${{github.event.inputs.tag}}
+ if [[ -z "${INPUT_TAG}" ]]; then
+ RESOURCE="built-from-latest-source"
+ else
+ RESOURCE=${{github.event.inputs.tag}}
+ fi
+ RECEIVER_LIST=${{secrets.FAPI_RECEIVER_LIST}}
+ if [[ -z "${RECEIVER_LIST}" ]]; then
+ RECEIVER_LIST=${{secrets.RECEIVER_LIST}}
+ fi
+ python3 ./product-is/oidc-conformance-tests/send_email.py $CONFORMANCE_SUITE_URL $GITHUB_RUN_NUMBER ${{job.status}} ${{github.repository}} ${{github.run_id}} ${{secrets.SENDER_EMAIL}} ${{secrets.PASSWORD}} ${{secrets.FAPI_RECEIVER_LIST}} $RESOURCE
+ elif [ $SEND_EMAIL == "NO" ]; then
+ echo "========================================"
+ echo "Skipped Sending Email"
+ echo "========================================"
+ else
+ echo "================================================================="
+ echo "Invalid parameter value. Skipped sending email"
+ echo "================================================================="
+ fi
  
+ - name: Send Chat message
+ if: always()
+ run: |
+ INPUT=${{github.event.inputs.send-chat}}
+ if [[ -z "${INPUT}" ]]; then
+ INPUT="yes"
+ fi
+
+ INPUT_TAG=${{github.event.inputs.tag}}
+ if [[ -z "${INPUT_TAG}" ]]; then
+ RESOURCE="built from latest source"
+ else
+ RESOURCE=${{github.event.inputs.tag}}
+ fi
+ 
+ SEND_CHAT=${INPUT^^}
+ if [ $SEND_CHAT == "YES" ]; then
+ echo "==========================="
+ echo "Sending Google Chat Message"
+ echo "==========================="
+ CONFORMANCE_SUITE_URL=https://localhost:8443
+ python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE" "FAPI OIDC Conformance Test"
+ elif [ $SEND_CHAT == "NO" ]; then
+ echo "========================================"
+ echo "Skipped Sending Google Chat Message"
+ echo "========================================"
+ else
+ echo "================================================================="
+ echo "Invalid parameter value. Skipped sending google chat message"
+ echo "================================================================="
+ fi
 

.github/workflows/oidc-conformance-test.yml

@@ -184,7 +184,7 @@ jobs:
  echo "Sending Google Chat Message"
  echo "==========================="
  CONFORMANCE_SUITE_URL=https://localhost:8443
- python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE"
+ python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE" "OIDC Conformance Test"
  elif [ $SEND_CHAT == "NO" ]; then
  echo "========================================"
  echo "Skipped Sending Google Chat Message"

oidc-conformance-tests/send_chat.py

@@ -27,6 +27,7 @@
 github_run_id = str(sys.argv[5])
 google_chat_webhook = sys.argv[6]
 wso2_is_version = sys.argv[7]
+workflow_name = sys.argv[8]
 
 failed_count = 0
 warnings_count = 0
@@ -56,7 +57,7 @@
  "cards": [
  {
  "header": {
- "title": "OIDC Conformance Test",
+ "title": workflow_name,
  "subtitle": "GitHub Action #" + github_run_number
 
  },
@@ -65,7 +66,7 @@
  "widgets": [
  {
  "textParagraph": {
- "text": f"<b>Identity Server {wso2_is_version} </b>"
+ "text": f"Identity Server version: <b>{wso2_is_version}</b>"
  }
  },
  {

oidc-fapi-conformance-tests/README.md

@@ -0,0 +1,32 @@
+This folder contains the configuration files and scripts that can be used to automate OIDC FAPI conformance testing. These tests can be run using GitHub actions or locally.
+
+## Testing using GitHub actions
+
+OIDC FAPI conformance test workflow can be used to for this purpose.
+1. Go to the Actions tab in the wso2/product-is github repository
+2. Click on FAPI OIDC Conformance Test workflow
+3. Click on Run workflow
+4. You need to provide the product-is release tag version you want to test (by default, it builds the latest IS by source)
+5. Also by default FAPI conformance suite is built from the latest released branch in https://gitlab.com/openid/conformance-suite.git. You can run against a specific conformance suite version also.
+6. Set 'Send test results to email' to 'yes' if you want to send test summary to a list of pre-configured email addresses. (default is 'no')
+7. Click on Run workflow
+8. After tests are completed you can view test results on the test summary page
+9. Two types of artifacts are saved after the test execution is completed
+ - test-logs - a log file is generated for each test plan. This log contains a summary of test cases with failures and warnings
+ - test-results - a zip file is generated for each test plan. You can use a web browser to view a detailed report of the test plan by extracting this zip file
+
+An email containing the same test summary will also be sent to a pre-configured list of email addresses. The sender email, password and the receiver emil list are the same as for OIDC action build. If you need to have a seperate receiver email list for FAPI Conformance action build,
+* Create a github secret with name `FAPI_RECEIVER_LIST` and add the list of receiver emails seperated by commas.
+
+Default configuration is to use Gmail SMTP server. You can change that by modifying `SMTP_SERVER` and `SMTP_SERVER_PORT` in `constants_fapi.py`
+
+This workflow is scheduled to run daily at 08:30 UTC (2:00 AM SL time) and will also automatically trigger after a release or a pre-release.
+
+To locally setup and run the test suite, follow the [fapi-oidc-conformance-test.yml](.github/workflows/fapi-oidc-conformance-test.yml) script and execute the steps.
+
+## Test Profiles
+
+Running fapi test profiles are in [test_runner_fapi.sh](oidc-fapi-conformance-tests/test_runner_fapi.sh) script. Currently Running profiles,
+* private_key_jwt
+* mtls
+

oidc-fapi-conformance-tests/configure_is_fapi.py

@@ -14,10 +14,6 @@
 from config import browser_configuration
 from config.client_configs import client_configs
 
-# path to product is zip file
-path_to_is_zip = str(sys.argv[1])
-print("Path to zip: ", path_to_is_zip)
-
 def decode_secret(secret):
  decoded_string=base64.b64decode(secret+"=").decode("utf-8")
  decoded_json = json.loads(decoded_string)
@@ -153,7 +149,10 @@ def addCertsToKeystore(rootCertPath, issuerCertPath, ISPath):
 
 
 # unpack product-is zip file and run
-def unpack_and_run(zip_file_name):
+def unpack_and_run():
+ # path to product is zip file
+ zip_file_name = str(sys.argv[1])
+ print("Path to zip: ", zip_file_name)
  try:
  # extract IS zip
  with ZipFile(zip_file_name, 'r') as zip_file:
@@ -261,7 +260,7 @@ def is_process_running(process_name):
 warnings.filterwarnings("ignore")
 
 if not is_process_running("wso2server"):
- unpack_and_run(path_to_is_zip)
+ unpack_and_run()
 else:
  print("\n>>> IS already running ...")
 print ("==============================================\n")

oidc-fapi-conformance-tests/constants_fapi.py

@@ -156,3 +156,8 @@
  "jwks_uri": JWKS_2,
  "require_pushed_authorization_requests" : "true",
 }
+
+
+SMTP_SERVER = "smtp.gmail.com"
+
+SMTP_SERVER_PORT = 465