Test RIO queue; command line and github action (#308)

* Test rio

* Test hourly, not on push

* Added docs

.github/workflows/test-rio.yml

@@ -0,0 +1,57 @@
+name: Check if RIO queue is up or down
+
+on:
+  schedule:
+    - cron: '6 9-17 * * 1-5' # every hour on weekdays during working hours
+
+jobs:
+  deps:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Cache lein project dependencies
+        uses: actions/cache@v4
+        with:
+          path: "~/.m2/repository"
+          key: "${{ runner.os }}-clojure-${{ hashFiles('**/project.clj') }}"
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup certificate stores
+        env:
+          KEYSTORE_JKS_B64: ${{ secrets.KEYSTORE_JKS_B64 }}
+          TRUSTSTORE_JKS_B64: ${{ secrets.TRUSTSTORE_JKS_B64 }}
+        run: |
+          echo "$KEYSTORE_JKS_B64" | base64 -d > keystore.jks
+          echo "$TRUSTSTORE_JKS_B64" | base64 -d > truststore.jks          
+
+      - name: Check rio queue status
+        env:
+          CLIENTS_INFO_PATH: ${{ secrets.CLIENTS_INFO_PATH }}
+          CLIENT_ID: ${{ secrets.CLIENT_ID }}
+          CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
+          GATEWAY_PASSWORD: ${{ secrets.GATEWAY_PASSWORD }}
+          GATEWAY_ROOT_URL: ${{ secrets.GATEWAY_ROOT_URL }}
+          GATEWAY_USER: ${{ secrets.GATEWAY_USER }}
+          KEYSTORE: keystore.jks
+          KEYSTORE_ALIAS: ${{ secrets.KEYSTORE_ALIAS }}
+          KEYSTORE_PASSWORD: ${{ secrets.KEYSTORE_PASSWORD }}
+          OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }}
+          OS_CONTAINER_NAME: ${{ secrets.OS_CONTAINER_NAME }}
+          OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
+          OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }}
+          OS_USERNAME: ${{ secrets.OS_USERNAME }}
+          RIO_READ_URL: ${{ secrets.RIO_READ_URL }}
+          RIO_RECIPIENT_OIN: ${{ secrets.RIO_RECIPIENT_OIN }}
+          RIO_SENDER_OIN: ${{ secrets.RIO_SENDER_OIN }}
+          RIO_RETRY_ATTEMPTS_SECONDS: '5,10,15'
+          RIO_UPDATE_URL: ${{ secrets.RIO_UPDATE_URL }}
+          STORE_HTTP_REQUESTS: true
+          SURF_CONEXT_CLIENT_ID: ${{ secrets.SURF_CONEXT_CLIENT_ID }}
+          SURF_CONEXT_CLIENT_SECRET: ${{ secrets.SURF_CONEXT_CLIENT_SECRET }}
+          SURF_CONEXT_INTROSPECTION_ENDPOINT: ${{ secrets.SURF_CONEXT_INTROSPECTION_ENDPOINT }}
+          TOKEN_ENDPOINT: ${{ secrets.TOKEN_ENDPOINT }}
+          TRUSTSTORE: truststore.jks
+          TRUSTSTORE_PASSWORD: ${{ secrets.TRUSTSTORE_PASSWORD }}
+        run: lein mapper test-rio rio-mapper-dev.jomco.nl

.gitignore

@@ -1,6 +1,7 @@
 /target
 /classes
 /checkouts
+/scripts
 /.idea
 *.iml
 profiles.clj

CLI.md

@@ -260,3 +260,14 @@ hostname (default localhost, port 8080).
 ### worker
 
 This action starts a worker.
+
+### test-rio
+
+This action tests if the RIO queue is up or down. If the queue is up, it will exit with code 0.
+If it is down, it will print a warning and exit with code -1.
+
+Example:
+
+```sh
+lein mapper test-rio uni-id
+```

src/nl/surf/eduhub_rio_mapper/cli_commands.clj

@@ -18,6 +18,7 @@
 
 (ns nl.surf.eduhub-rio-mapper.cli-commands
   (:require [clojure.data.json :as json]
+            [clojure.java.io :as io]
             [clojure.string :as str]
             [nl.surf.eduhub-rio-mapper.clients-info :as clients-info]
             [nl.surf.eduhub-rio-mapper.endpoints.api :as api]
@@ -26,9 +27,10 @@
             [nl.surf.eduhub-rio-mapper.rio.loader :as rio.loader]
             [nl.surf.eduhub-rio-mapper.specs.ooapi :as ooapi]
             [nl.surf.eduhub-rio-mapper.specs.rio :as rio]
-            [nl.surf.eduhub-rio-mapper.worker :as worker]))
+            [nl.surf.eduhub-rio-mapper.worker :as worker])
+  (:import [java.util UUID]))
 
-(defn parse-getter-args [[type id & [pagina]]]
+(defn- parse-getter-args [[type id & [pagina]]]
   {:pre [type id (string? type)]}
   (let [[type response-type] (reverse (str/split type #":" 2))
         response-type (and response-type (keyword response-type))
@@ -56,7 +58,7 @@
       (System/exit 1))
     [client-info rest-args]))
 
-(defn process-command [command args {{:keys [getter resolver ooapi-loader dry-run! link!] :as handlers} :handlers {:keys [clients] :as config} :config}]
+(defn process-command [command args {{:keys [getter resolver ooapi-loader dry-run! link! insert!] :as handlers} :handlers {:keys [clients] :as config} :config}]
   {:pre [getter]}
   (case command
     "serve-api"
@@ -70,6 +72,30 @@
       (worker/wait-worker
         (worker/start-worker! config)))
 
+    "test-rio"
+    (let [[client-info _] (parse-client-info-args args clients)
+          uuid    (UUID/randomUUID)
+          eduspec (-> "../test/fixtures/ooapi/education-specification-template.json"
+                      io/resource
+                      slurp
+                      (json/read-str :key-fn keyword)
+                      (assoc :educationSpecificationId uuid))]
+
+      (try
+        (insert! {:institution-oin        (:institution-oin client-info)
+                  :institution-schac-home (:institution-schac-home client-info)
+                  ::ooapi/type            "education-specification"
+                  ::ooapi/id              uuid
+                  ::ooapi/entity          eduspec})
+        (println "The RIO Queue is UP")
+        (catch Exception ex
+          (when-let [ex-data (ex-data ex)]
+            (when (= :down (:rio-queue-status ex-data))
+              (println "The RIO Queue is DOWN")
+              (System/exit -1)))
+          (println "An unexpected exception has occurred: " ex)
+          (System/exit -2))))
+
     "get"
     (let [[client-info rest-args] (parse-client-info-args args clients)]
       (getter (assoc (parse-getter-args rest-args)

src/nl/surf/eduhub_rio_mapper/commands/processing.clj

@@ -128,7 +128,7 @@
          :eduspec       eduspec
          :mutate-result mutate-result})
       (throw (ex-info (str "Processing this job takes longer than expected. Our developers have been informed and will contact DUO. Please try again in a few hours."
-                           ": " type " " id) {})))))
+                           ": " type " " id) {:rio-queue-status :down})))))
 
 (defn- make-updater-sync-relations-phase [handlers]
   (fn sync-relations-phase [{:keys [job eduspec] :as request}]
@@ -145,6 +145,18 @@
                         (assoc (ex-data ex) :phase phase)
                         ex))))))
 
+(defn- make-insert [handlers rio-config]
+  (let [fs [[:preparing      (make-updater-soap-phase)]
+            [:upserting      (make-updater-mutate-rio-phase handlers)]
+            [:confirming     (make-updater-confirm-rio-phase handlers rio-config)]]
+        wrapped-fs (map wrap-phase fs)]
+    (fn [request]
+      {:pre [(:institution-oin request)
+             (::ooapi/entity request)]}
+      (as-> request $
+            (reduce (fn [req f] (f req)) $ wrapped-fs)
+            (:mutate-result $)))))
+
 (defn- make-update [handlers rio-config]
   (let [fs [[:fetching-ooapi (make-updater-load-ooapi-phase handlers)]
             [:resolving      (make-updater-resolve-phase handlers)]
@@ -237,6 +249,7 @@
                       :resolver     resolver}
         update!      (make-update handlers rio-config)
         delete!      (make-deleter handlers)
+        insert!      (make-insert handlers rio-config)
         dry-run!     (make-dry-runner handlers)
         link!        (link/make-linker rio-config getter)]
-    (assoc handlers :update! update!, :delete! delete!, :dry-run! dry-run!, :link! link!)))
+    (assoc handlers :update! update!, :delete! delete!, :insert! insert!, :dry-run! dry-run!, :link! link!)))

src/nl/surf/eduhub_rio_mapper/main.clj

@@ -26,7 +26,7 @@
   (:gen-class))
 
 (def commands
-  #{"upsert" "delete" "delete-by-code" "get" "show" "resolve" "serve-api" "worker" "help" "dry-run-upsert" "link"})
+  #{"upsert" "delete" "delete-by-code" "get" "show" "resolve" "serve-api" "worker" "help" "dry-run-upsert" "link" "test-rio"})
 
 (defn -main
   [command & args]
@@ -53,7 +53,7 @@
       ("dry-run-upsert" "link")
       (pprint/pprint result)
 
-      "show"
+      ("show" "test-rio")
       nil
 
       "resolve"

src/nl/surf/eduhub_rio_mapper/rio/loader.clj

@@ -151,7 +151,6 @@
   (fn resolver
     [type id institution-oin]
     {:pre [institution-oin]}
-
     (when id
       (logging/with-mdc
         {:soap-action "opvragen_rioIdentificatiecode" :ooapi-id id}

test/fixtures/ooapi/education-specification-template.json

@@ -0,0 +1,63 @@
+{
+  "educationSpecificationId": "123e4567-e89b-12d3-a456-426614174000",
+  "primaryCode": {
+    "codeType": "crohoCreboCode",
+    "code": "string"
+  },
+  "otherCodes": [
+    {
+      "codeType": "crohoCreboCode",
+      "code": "1234123"
+    }
+  ],
+  "educationSpecificationType": "program",
+  "name": [
+    {
+      "language": "en-GB",
+      "value": "Bachelor Chemical technology"
+    },
+    {
+      "language": "nl-BE",
+      "value": "Bachelor Chemische technologie"
+    }
+  ],
+  "abbreviation": "B Scheikundige Technologie",
+  "description": [
+    {
+      "language": "en-GB",
+      "value": "program that is a place holder for all courses that are made available for student mobility, program that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobility"
+    }
+  ],
+  "formalDocument": "diploma",
+  "level": "master",
+  "sector": "university education",
+  "levelOfQualification": "6",
+  "fieldsOfStudy": "0732",
+  "studyLoad": {
+    "studyLoadUnit": "ects",
+    "value": 3
+  },
+  "learningOutcomes": [
+    [
+      {
+        "language": "en-GB",
+        "value": "Executable knowledge of Chemical technology, including: Acquire knowledge of research paradigms."
+      }
+    ]
+  ],
+  "link": "https://bijvak.nl",
+  "parent": "42e2bc1b-6741-4e2e-b138-97b4a342c999",
+  "children": [
+    "497f6eca-6276-4993-bfeb-53cbbbba6f08"
+  ],
+  "organization": "452c1a86-a0af-475b-b03f-724878b0f387",
+  "consumers": [
+    {
+      "consumerKey": "rio",
+      "educationSpecificationSubType": "variant"
+    }
+  ],
+  "ext": {},
+  "validFrom": "2019-08-24",
+  "validTo": "2019-08-25"
+}