diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3b768fb8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +/*.jar +/target diff --git a/.gitignore b/.gitignore index af0033b5..2323a6c2 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ profiles.clj .DS_Store pom.xml pom.xml.asc -*.jar *.class /.lein-* /.nrepl-port @@ -19,3 +18,4 @@ pom.xml.asc /logs /keystore.jks /.envrc +/.envrc-docker diff --git a/Dockerfile b/Dockerfile index 8c8c6656..e7db7cf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM clojure:temurin-17-lein-2.11.2-alpine as builder +FROM clojure:temurin-17-lein-2.11.2-alpine AS builder RUN mkdir /app WORKDIR /app @@ -7,5 +7,9 @@ RUN lein uberjar FROM gcr.io/distroless/java17-debian12 COPY --from=builder /app/target/eduhub-rio-mapper.jar /eduhub-rio-mapper.jar +# Make sure there is an opentelemetry agent in the workdir in case docker-compose +# starts up a process with -javaagent in the JAVA_TOOL_OPTIONS +COPY --from=builder /app/vendor/opentelemetry-javaagent-2.9.0.jar /opentelemetry-javaagent.jar +WORKDIR / ENTRYPOINT ["java", "-jar", "/eduhub-rio-mapper.jar"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..bcef81b2 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +.PHONY: lint test check clean + +lint: + clojure -M:lint + +test: + clojure -M:test + +check: lint test + +clean: + rm -rf classes target + +.PHONY: test lint check diff --git a/dev-infra/prometheus.yml b/dev-infra/prometheus.yml new file mode 100644 index 00000000..140a2637 --- /dev/null +++ b/dev-infra/prometheus.yml @@ -0,0 +1,32 @@ +global: + scrape_interval: 15s + evaluation_interval: 30s + + external_labels: + monitor: codelab + foo: bar + +scrape_configs: + - job_name: scrape-static + + honor_labels: true + # scrape_interval is defined by the configured global (15s). + # scrape_timeout is defined by the global default (10s). + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + scheme: http + + static_configs: + - targets: ["localhost:9090"] + labels: + service: prometheus + - targets: ["worker:9464"] + labels: + service: worker + - targets: ["api:3000"] + labels: + service: api + - targets: ["api:9464"] + labels: + service: api diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..3e7cd39a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,63 @@ +services: + redis: + image: redis:7-alpine + restart: always + ports: + - '6380:6379' + command: redis-server --save 20 1 --loglevel warning + volumes: + - redis:/data + prometheus: + image: prom/prometheus + ports: + - "9090:9090" + volumes: + - ./dev-infra/prometheus.yml:/etc/prometheus/prometheus.yml + worker: + platform: "linux/amd64" + build: . + env_file: + - .envrc # read environment from local direnv settings + environment: + CLIENTS_INFO_PATH: /test/test-clients.json + REDIS_URI: redis://redis + OTEL_METRICS_EXPORTER: prometheus + OTEL_EXPORTER_PROMETHEUS_ENDPOINT: http://localhost:9464/metrics + OTEL_SERVICE_NAME: edumapper-worker + OTEL_LOGS_EXPORTER: none + OTEL_TRACES_EXPORTER: none + JAVA_TOOL_OPTIONS: -javaagent:./opentelemetry-javaagent.jar + volumes: + - ./truststore.jks:/truststore.jks + - ./keystore.jks:/keystore.jks + - ./test/test-clients.json:/test/test-clients.json + command: worker + ports: + - "9465:9464" + api: + platform: "linux/amd64" + build: . + env_file: + - .envrc # read environment from local direnv settings + environment: + CLIENTS_INFO_PATH: /test/test-clients.json + REDIS_URI: redis://redis + OTEL_METRICS_EXPORTER: prometheus + OTEL_EXPORTER_PROMETHEUS_ENDPOINT: http://localhost:9464/metrics + OTEL_SERVICE_NAME: edumapper-api + OTEL_LOGS_EXPORTER: none + OTEL_TRACES_EXPORTER: none + JAVA_TOOL_OPTIONS: -javaagent:./opentelemetry-javaagent.jar + API_HOSTNAME: 0.0.0.0 + API_PORT: 3000 + volumes: + - ./truststore.jks:/truststore.jks + - ./keystore.jks:/keystore.jks + - ./test/test-clients.json:/test/test-clients.json + ports: + - "3000:3000" + - "9464:9464" + command: serve-api +volumes: + redis: + driver: local diff --git a/project.clj b/project.clj index ace0c7bd..eca1b0ae 100644 --- a/project.clj +++ b/project.clj @@ -3,7 +3,8 @@ :url "https://github.com/jomco/eduhub-rio-mapper" :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} - :dependencies [[org.clojure/clojure "1.12.0"] + :dependencies [[com.github.steffan-westcott/clj-otel-api "0.2.7"] + [org.clojure/clojure "1.12.0"] [org.clojure/core.async "1.6.681"] [org.clojure/core.memoize "1.1.266"] [com.velisco/strgen "0.2.5" :exclusions [org.clojure/clojurescript]] @@ -54,6 +55,8 @@ [nl.jomco/proof-specs "RELEASE"] [ring/ring-mock "RELEASE"]] :plugins [[lein-ancient "RELEASE"]] + ;; Uncomment this to test the opentelemetry agent + ;;:jvm-opts ["-javaagent:vendor/opentelemetry-javaagent-2.9.0.jar"] :aliases {"lint" ["run" "-m" "clj-kondo.main" "--lint" "src" "test"] "check-deps" ["ancient" "check" ":no-profiles" ":exclude" "keep-this-version"] "upgrade-deps" ["ancient" "upgrade" ":no-profiles" ":exclude" "keep-this-version"] diff --git a/src/nl/surf/eduhub_rio_mapper/config.clj b/src/nl/surf/eduhub_rio_mapper/config.clj index 41e84588..da9e9934 100644 --- a/src/nl/surf/eduhub_rio_mapper/config.clj +++ b/src/nl/surf/eduhub_rio_mapper/config.clj @@ -23,9 +23,11 @@ [nl.jomco.envopts :as envopts] [nl.surf.eduhub-rio-mapper.clients-info :as clients-info] [nl.surf.eduhub-rio-mapper.commands.processing :as processing] + [nl.surf.eduhub-rio-mapper.endpoints.metrics :as metrics] [nl.surf.eduhub-rio-mapper.endpoints.status :as status] [nl.surf.eduhub-rio-mapper.job :as job] - [nl.surf.eduhub-rio-mapper.utils.keystore :as keystore])) + [nl.surf.eduhub-rio-mapper.utils.keystore :as keystore] + [nl.surf.eduhub-rio-mapper.worker :as worker])) (defn parse-int-list [s & _opts] [(mapv #(Integer/parseInt %) (str/split s #","))]) @@ -158,14 +160,26 @@ trust-store-pass)) (assoc :clients (clients-info/read-clients-data clients-info-config))))))) -(defn make-config-and-handlers [] +(defn make-config-and-handlers-web [] (let [{:keys [clients] :as cfg} (make-config) handlers (processing/make-handlers cfg) config (update cfg :worker merge {:queues (clients-info/institution-schac-homes clients) - :queue-fn :institution-schac-home - :run-job-fn #(job/run! handlers % (= (System/getenv "STORE_HTTP_REQUESTS") "true")) - :set-status-fn (status/make-set-status-fn cfg) - :retryable-fn status/retryable? - :error-fn status/errors?})] + :queue-fn :institution-schac-home})] + {:handlers handlers :config config})) + +(defn make-config-and-handlers-worker [] + (let [{:keys [clients] :as cfg} (make-config) + handlers (processing/make-handlers cfg) + schac-home-to-name (reduce (fn [h c] (assoc h (:institution-schac-home c) (:institution-name c))) {} clients) + institution-schac-homes (clients-info/institution-schac-homes clients) + config (update cfg :worker merge + {:queues (clients-info/institution-schac-homes clients) + :queue-fn :institution-schac-home + :run-job-fn #(job/run! handlers % (= (System/getenv "STORE_HTTP_REQUESTS") "true")) + :set-status-fn (status/make-set-status-fn cfg) + :retryable-fn status/retryable? + :error-fn status/errors? + ;; The web-api doesn't need the job-counter + :jobs-counter-fn (metrics/make-jobs-counter schac-home-to-name #(worker/queue-counts-by-key % cfg) institution-schac-homes)})] {:handlers handlers :config config})) diff --git a/src/nl/surf/eduhub_rio_mapper/endpoints/api.clj b/src/nl/surf/eduhub_rio_mapper/endpoints/api.clj index 1ee4ec5f..d0955e35 100644 --- a/src/nl/surf/eduhub_rio_mapper/endpoints/api.clj +++ b/src/nl/surf/eduhub_rio_mapper/endpoints/api.clj @@ -23,10 +23,9 @@ [compojure.route :as route] [nl.jomco.http-status-codes :as http-status] [nl.jomco.ring-trace-context :refer [wrap-trace-context]] - [nl.surf.eduhub-rio-mapper.clients-info :refer [wrap-client-info] :as clients-info] + [nl.surf.eduhub-rio-mapper.clients-info :refer [wrap-client-info]] [nl.surf.eduhub-rio-mapper.endpoints.app-server :as app-server] [nl.surf.eduhub-rio-mapper.endpoints.health :as health] - [nl.surf.eduhub-rio-mapper.endpoints.metrics :as metrics] [nl.surf.eduhub-rio-mapper.endpoints.status :as status] [nl.surf.eduhub-rio-mapper.job :as job] [nl.surf.eduhub-rio-mapper.specs.ooapi :as ooapi-specs] @@ -86,15 +85,6 @@ (update res :job assoc ::job/callback-url callback-url) {:status http-status/bad-request :body "Malformed callback url"}))))) -(defn wrap-metrics-getter - [app count-queues-fn fetch-jobs-by-status schac-home-to-name] - (fn with-metrics-getter [req] - (let [res (app req)] - (cond-> res - (:metrics res) - (assoc :status http-status/ok - :body (metrics/prometheus-render-metrics (count-queues-fn) (fetch-jobs-by-status) schac-home-to-name)))))) - (defn wrap-status-getter [app {:keys [status-getter-fn]}] (fn status-getter [req] @@ -199,10 +189,8 @@ (def public-routes (compojure.core/routes - (GET "/metrics" [] - {:metrics true}) - (GET "/health" [] - {:health true}))) + (GET "/health" [] + {:health true}))) (defn read-only-routes [config] (-> (GET "/status/:token" [token] {:token token}) @@ -218,18 +206,11 @@ (route/not-found nil))) (defn make-app [{:keys [auth-config clients] :as config}] - (let [institution-schac-homes (clients-info/institution-schac-homes clients) - schac-home-to-name (reduce (fn [h c] (assoc h (:institution-schac-home c) (:institution-name c))) {} clients) - queue-counter-fn (fn [] (metrics/count-queues #(worker/queue-counts-by-key % config) institution-schac-homes)) - jobs-by-status-counter-fn (fn [] (metrics/fetch-jobs-by-status-count config)) - token-authenticator (-> (authentication/make-token-authenticator auth-config) + (let [token-authenticator (-> (authentication/make-token-authenticator auth-config) (authentication/cache-token-authenticator {:ttl-minutes 10}))] (-> (routes {:enqueuer-fn (partial worker/enqueue! config) :status-getter-fn (partial status/rget config)}) (health/wrap-health config) - (wrap-metrics-getter queue-counter-fn - jobs-by-status-counter-fn - schac-home-to-name) (wrap-client-info clients) (authentication/wrap-authentication token-authenticator) (wrap-logging) diff --git a/src/nl/surf/eduhub_rio_mapper/endpoints/metrics.clj b/src/nl/surf/eduhub_rio_mapper/endpoints/metrics.clj index 2479d4cc..6d7f20a2 100644 --- a/src/nl/surf/eduhub_rio_mapper/endpoints/metrics.clj +++ b/src/nl/surf/eduhub_rio_mapper/endpoints/metrics.clj @@ -17,66 +17,41 @@ ;; . (ns nl.surf.eduhub-rio-mapper.endpoints.metrics - (:require [clojure.string :as str] - [nl.surf.eduhub-rio-mapper.utils.redis :as redis])) + (:require [clojure.string] + [steffan-westcott.clj-otel.api.metrics.instrument :as instrument])) (def jobs-count-by-status-key-name "jobs-count-by-status") -(defn increment-count [{:keys [redis-conn]} job status] - (let [hash-key (str (:institution-schac-home job) "/" (name status))] - (redis/hincrby redis-conn jobs-count-by-status-key-name hash-key 1))) +(declare count-queues) + +(defn- update-gauge [gauge data] + (doseq [[k v] data] + (instrument/set! gauge {:value v :attributes {"queue-name" k}}))) + +(defn make-jobs-counter [schac-home-to-name queue-counter institution-schac-homes] + (let [counter (instrument/instrument {:name "rio_mapper_http_requests_total" + :instrument-type :counter}) + + gauge (instrument/instrument {:name "rio_mapper_active_and_queued_job_count" + :instrument-type :gauge})] + (fn [job status] + (update-gauge gauge (count-queues queue-counter institution-schac-homes)) + (let [schac-home (:institution-schac-home job) + attributes {:status (name status) + :schac-home schac-home + :institution-name (schac-home-to-name schac-home)}] + (instrument/add! counter {:value 1 :attributes attributes}))))) ;; Wraps the set-status-fn in order to increment the job count if it is a final status ;; (done,error,timeout). These, and only these, have a third argument with the result. -(defn wrap-increment-count [config set-status-fn] +(defn wrap-increment-count [jobs-counter set-status-fn] (fn ([job status] (set-status-fn job status)) ([job status result] - (increment-count config job status) + (jobs-counter job status) (set-status-fn job status result)))) -;; Retrieves the total number of processed jobs by status (started,done,error,time_out) -;; The job count is grouped per schachome. Returns a map with as keys the status (keyword) -;; and as values maps with as keys the schachome and count (integer) as values. -(defn fetch-jobs-by-status-count [{:keys [redis-conn] :as _config}] - {:post [(every? string? (keys %)) - (every? map? (vals %))]} - (let [process-pair - (fn [[k cnt]] - (let [[schach-home status] (clojure.string/split k #"/")] - [status schach-home cnt])) - process-triplet - (fn [h [status schach-home cnt]] - (assoc-in h [status schach-home] cnt))] - (->> (redis/hgetall redis-conn jobs-count-by-status-key-name) - (partition 2) - (map process-pair) - (reduce process-triplet {})))) - -(defn prometheus-current-jobs [queue-count schac-home-to-name] - (map (fn [[k v]] (format "rio_mapper_active_and_queued_job_count{schac_home=\"%s\", institution_name=\"%s\"} %s" k (schac-home-to-name k) v)) - queue-count)) - -(defn prometheus-jobs-by-status [jobs-count-by-status schac-home-to-name] - {:pre [(every? string? (keys jobs-count-by-status)) - (every? map? (vals jobs-count-by-status))]} - (mapcat (fn [status] - (map (fn [[k v]] - (format "rio_mapper_jobs_total{schac_home=\"%s\", institution_name=\"%s\", job_status=\"%s\"} %s" k (schac-home-to-name k) status v)) - (get jobs-count-by-status status))) - ["started" "done" "time_out" "error"])) - -(defn prometheus-render-metrics [current-queue-count jobs-count-by-status schac-home-to-name] - {:pre [(map? current-queue-count) - (every? string? (keys current-queue-count)) - (every? integer? (vals current-queue-count)) - (map? jobs-count-by-status) - (every? string? (keys jobs-count-by-status)) - (every? map? (vals jobs-count-by-status))]} - (str/join "\n" (into (prometheus-current-jobs current-queue-count schac-home-to-name) - (prometheus-jobs-by-status jobs-count-by-status schac-home-to-name)))) - (defn count-queues [grouped-queue-counter client-schac-homes] {:post [(map? %) (every? string? (keys %)) diff --git a/src/nl/surf/eduhub_rio_mapper/main.clj b/src/nl/surf/eduhub_rio_mapper/main.clj index 21f74ef0..7a07c383 100644 --- a/src/nl/surf/eduhub_rio_mapper/main.clj +++ b/src/nl/surf/eduhub_rio_mapper/main.clj @@ -41,7 +41,9 @@ (println (config/help)) (System/exit 0)) - (let [result (cli-commands/process-command command args (config/make-config-and-handlers))] + (let [result (cli-commands/process-command command args (if (= command "worker") + (config/make-config-and-handlers-worker) + (config/make-config-and-handlers-web)))] (case command ("serve-api" "worker") nil diff --git a/src/nl/surf/eduhub_rio_mapper/worker.clj b/src/nl/surf/eduhub_rio_mapper/worker.clj index de13eb73..38acb93b 100644 --- a/src/nl/surf/eduhub_rio_mapper/worker.clj +++ b/src/nl/surf/eduhub_rio_mapper/worker.clj @@ -188,8 +188,11 @@ - `error-fn` a function which takes one argument, the result of the job, and returns true when the job failed. - - `retryable-fn` a functions which one argument, the result of the + - `retryable-fn` a function which takes one argument, the result of the job, and returns true when job failed but can be retried. + + - `jobs-counter-fn` a function which takes two arguments, a job and a job status, + and updates the metrics related to the job status type. Also updates the metrics for the queues. " [{{:keys [queues lock-ttl-ms @@ -202,7 +205,8 @@ error-fn retryable-fn run-job-fn - set-status-fn] + set-status-fn + jobs-counter-fn] ;; Set lock expiry to 1 minute; locks in production have unexpectedly expired with shorter intervals :or {lock-ttl-ms 60000 nap-ms 1000}} :worker @@ -210,6 +214,7 @@ stop-atom] {:pre [retry-wait-ms max-retries + jobs-counter-fn (seq queues) (fn? run-job-fn) (fn? set-status-fn) (ifn? retryable-fn) (ifn? error-fn) (ifn? queue-fn)]} @@ -231,9 +236,9 @@ (str (Instant/now))))] ;; Don't count job as started while retrying it (when (nil? (::retries job)) - (metrics/increment-count config job :started)) + (jobs-counter-fn job :started)) ;; run job asynchronous - (let [set-status-fn (metrics/wrap-increment-count config set-status-fn) + (let [set-status-fn (metrics/wrap-increment-count jobs-counter-fn set-status-fn) c (async/thread (.setName (Thread/currentThread) (str "runner-" queue)) (run-job-fn job))] diff --git a/test/nl/surf/eduhub_rio_mapper/endpoints/api_test.clj b/test/nl/surf/eduhub_rio_mapper/endpoints/api_test.clj index 7adbe96f..fcf14e37 100644 --- a/test/nl/surf/eduhub_rio_mapper/endpoints/api_test.clj +++ b/test/nl/surf/eduhub_rio_mapper/endpoints/api_test.clj @@ -205,20 +205,6 @@ (is (= res (app res)) "valid opleidingscode")))) -(deftest metrics - (let [app (api/wrap-metrics-getter - api-routes - (constantly {"foo" 1, "bar" 2}) - (constantly {"done" {"google" 123, "meta" 321}}) - {"delft" "TU", "leiden" "LU", "google" "alphabet", "meta" "facebook", "foo" "OOF", "bar" "RAB"}) - {:keys [status body]} (app (request :get "/metrics"))] - (is (= http-status/ok status)) - (is (= (str "rio_mapper_jobs_total{schac_home=\"meta\", institution_name=\"facebook\", job_status=\"done\"} 321\n" - "rio_mapper_jobs_total{schac_home=\"google\", institution_name=\"alphabet\", job_status=\"done\"} 123\n" - "rio_mapper_active_and_queued_job_count{schac_home=\"foo\", institution_name=\"OOF\"} 1\n" - "rio_mapper_active_and_queued_job_count{schac_home=\"bar\", institution_name=\"RAB\"} 2") - body)))) - (deftest wrap-job-queuer (let [queue-atom (atom []) app (api/wrap-job-enqueuer identity #(swap! queue-atom conj %))] diff --git a/test/nl/surf/eduhub_rio_mapper/metrics_test.clj b/test/nl/surf/eduhub_rio_mapper/metrics_test.clj deleted file mode 100644 index 8fbdb0c5..00000000 --- a/test/nl/surf/eduhub_rio_mapper/metrics_test.clj +++ /dev/null @@ -1,30 +0,0 @@ -;; This file is part of eduhub-rio-mapper -;; -;; Copyright (C) 2022 SURFnet B.V. -;; -;; This program is free software: you can redistribute it and/or -;; modify it under the terms of the GNU Affero General Public License -;; as published by the Free Software Foundation, either version 3 of -;; the License, or (at your option) any later version. -;; -;; This program is distributed in the hope that it will be useful, but -;; WITHOUT ANY WARRANTY; without even the implied warranty of -;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -;; Affero General Public License for more details. -;; -;; You should have received a copy of the GNU Affero General Public -;; License along with this program. If not, see -;; . - -(ns nl.surf.eduhub-rio-mapper.metrics-test - (:require [clojure.test :refer :all] - [nl.surf.eduhub-rio-mapper.endpoints.metrics :as metrics])) - -(deftest render-metrics - (is (= (str "rio_mapper_jobs_total{schac_home=\"meta\", institution_name=\"facebook\", job_status=\"done\"} 321\n" - "rio_mapper_jobs_total{schac_home=\"google\", institution_name=\"alphabet\", job_status=\"done\"} 123\n" - "rio_mapper_active_and_queued_job_count{schac_home=\"delft\", institution_name=\"TU\"} 12\n" - "rio_mapper_active_and_queued_job_count{schac_home=\"leiden\", institution_name=\"LU\"} 32") - (metrics/prometheus-render-metrics {"delft" 12 "leiden" 32} - {"done" {"google" 123, "meta" 321}} - {"delft" "TU", "leiden" "LU", "google" "alphabet", "meta" "facebook"})))) diff --git a/test/nl/surf/eduhub_rio_mapper/worker_test.clj b/test/nl/surf/eduhub_rio_mapper/worker_test.clj index fbe65440..81cf3ef4 100644 --- a/test/nl/surf/eduhub_rio_mapper/worker_test.clj +++ b/test/nl/surf/eduhub_rio_mapper/worker_test.clj @@ -27,14 +27,15 @@ {:redis-conn {:pool {} :spec {:uri (or (System/getenv "REDIS_URI") "redis://localhost")}} :redis-key-prefix "eduhub-rio-mapper-test" :status-ttl-sec 10 - :worker {:nap-ms 10 - :retry-wait-ms 10 - :max-retries 3 - :queues ["foo" "bar"] - :queue-fn :queue - :retryable-fn (constantly false) - :error-fn (constantly false) - :set-status-fn (fn [_ _ & [_]] (comment "nop"))}}) + :worker {:nap-ms 10 + :retry-wait-ms 10 + :max-retries 3 + :queues ["foo" "bar"] + :queue-fn :queue + :retryable-fn (constantly false) + :error-fn (constantly false) + :jobs-counter-fn (constantly nil) + :set-status-fn (fn [_ _ & [_]] (comment "nop"))}}) (deftest ^:redis worker (let [job-runs (atom {"foo" [], "bar" []}) @@ -79,6 +80,7 @@ max-retries 3 config (-> config (assoc-in [:worker :max-retries] max-retries) + (assoc-in [:worker :jobs-counter-fn] (constantly nil)) (assoc-in [:worker :run-job-fn] (fn [job] (reset! last-seen-job (dissoc job :started-at)) @@ -107,6 +109,7 @@ max-retries 3 config (-> config (assoc-in [:worker :max-retries] max-retries) + (assoc-in [:worker :jobs-counter-fn] (constantly nil)) (assoc-in [:worker :run-job-fn] (fn [job] (reset! last-seen-job (dissoc job :started-at)) @@ -135,6 +138,7 @@ retry-wait-ms 3000 config (-> config (assoc-in [:worker :retry-wait-ms] retry-wait-ms) + (assoc-in [:worker :jobs-counter-fn] (constantly nil)) (assoc-in [:worker :run-job-fn] (fn [job] (reset! last-seen-job (dissoc job :started-at)) @@ -168,6 +172,7 @@ config (-> config (assoc-in [:worker :error-fn] :error?) (assoc-in [:worker :run-job-fn] identity) + (assoc-in [:worker :jobs-counter-fn] (constantly nil)) (assoc-in [:worker :set-status-fn] (fn [job status & [data]] (reset! last-seen-status {:job (dissoc job :started-at) diff --git a/vendor/opentelemetry-javaagent-2.9.0.jar b/vendor/opentelemetry-javaagent-2.9.0.jar new file mode 100644 index 00000000..0a60e254 Binary files /dev/null and b/vendor/opentelemetry-javaagent-2.9.0.jar differ