From 2dcf6273493e2aca6808debe81f6a1b87826d70b Mon Sep 17 00:00:00 2001 From: Ilmari Vacklin Date: Mon, 25 Nov 2024 20:53:47 +0200 Subject: [PATCH] feat: OpenTelemetry --- .github/renovate.json | 8 + .mise.toml | 2 - docker-compose.yml | 15 ++ infra/lib/service-stack.ts | 13 ++ otel-config.yml | 34 +++ scripts/start_local_server.sh | 4 +- server/pom.xml | 20 ++ .../main/kotlin/fi/oph/kitu/ErrorHandler.kt | 10 +- .../fi/oph/kitu/csvparsing/CsvParser.kt | 28 +-- .../KoealustaScheduledTasks.kt | 4 +- .../kotoutumiskoulutus/KoealustaService.kt | 94 ++++----- .../logging/AccessLoggingConfiguration.kt | 34 --- .../fi/oph/kitu/logging/CloudWatchAppender.kt | 12 +- .../oppijanumero/CasAuthenticatedService.kt | 27 +-- .../fi/oph/kitu/oppijanumero/CasService.kt | 13 +- .../kitu/oppijanumero/OppijanumeroService.kt | 126 ++++++------ .../fi/oph/kitu/yki/YkiScheduledTasks.kt | 4 +- .../main/kotlin/fi/oph/kitu/yki/YkiService.kt | 194 ++++++++---------- .../fi/oph/kitu/yki/YkiViewController.kt | 3 + .../yki/arvioijat/YkiArvioijaRepository.kt | 2 + .../resources/application-prod.properties | 2 + .../main/resources/application-qa.properties | 2 + .../resources/application-untuva.properties | 2 + .../src/main/resources/application.properties | 8 +- server/src/main/resources/logback-spring.xml | 5 - .../fi/oph/kitu/csvparsing/CsvParsingTest.kt | 20 +- .../oppijanumero/OppijanumeroServiceTests.kt | 6 + .../kotlin/fi/oph/kitu/yki/YkiServiceTests.kt | 8 +- .../src/test/resources/application.properties | 9 +- 29 files changed, 347 insertions(+), 362 deletions(-) create mode 100644 otel-config.yml delete mode 100644 server/src/main/kotlin/fi/oph/kitu/logging/AccessLoggingConfiguration.kt diff --git a/.github/renovate.json b/.github/renovate.json index 228dceed..0b2f680a 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -29,6 +29,14 @@ "datasourceTemplate": "github-tags", "packageNameTemplate": "aws/aws-cli", "autoReplaceStringTemplate": "{{depName}} = \"ref:{{newVersion}}\"" + }, + { + "customType": "regex", + "fileMatch": ["\\.ts$"], + "matchStrings": [ + "renovate: datasource=(?.+?)\\s+\"(?\\S+):(?\\S+)\"" + ], + "versioningTemplate": "semver" } ] } diff --git a/.mise.toml b/.mise.toml index feadd91a..82f04e00 100644 --- a/.mise.toml +++ b/.mise.toml @@ -14,6 +14,4 @@ awscli = "ref:2.22.32" java = "corretto-21.0.5.11.1" shellcheck = "0.10.0" ktlint = "1.5.0" -go = "1.23.4" "npm:esbuild" = "0.24.0" -"go:github.com/humanlogio/humanlog/cmd/humanlog" = "0.7.8" diff --git a/docker-compose.yml b/docker-compose.yml index 35c90ec8..0c08597c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,3 +11,18 @@ services: PGUSER: kitu ports: - "127.0.0.1:5432:5432" + + collector: + image: otel/opentelemetry-collector:0.114.0 + command: ["--config=/etc/otel-collector-config.yml"] + volumes: + - ./otel-config.yml:/etc/otel-collector-config.yml + ports: + - "4318:4318" # OTLP HTTP + depends_on: + - jaeger + + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - "16686:16686" # UI diff --git a/infra/lib/service-stack.ts b/infra/lib/service-stack.ts index 17e356f1..02ee3bd4 100644 --- a/infra/lib/service-stack.ts +++ b/infra/lib/service-stack.ts @@ -19,6 +19,7 @@ import { DatabaseCluster } from "aws-cdk-lib/aws-rds" import { HostedZone } from "aws-cdk-lib/aws-route53" import { ITopic } from "aws-cdk-lib/aws-sns" import { Construct } from "constructs" +import { ManagedPolicy } from "aws-cdk-lib/aws-iam" export interface ServiceStackProps extends StackProps { auditLogGroup: ILogGroup @@ -135,6 +136,13 @@ export class ServiceStack extends Stack { sslPolicy: SslPolicy.RECOMMENDED_TLS, }) + this.service.taskDefinition.addContainer("AwsOtelCollector", { + image: ContainerImage.fromRegistry( + // renovate: datasource=docker + "public.ecr.aws/aws-observability/aws-otel-collector:v0.41.1", + ), + }) + this.service.targetGroup.configureHealthCheck({ ...this.service.targetGroup.healthCheck, path: "/actuator/health", @@ -142,6 +150,11 @@ export class ServiceStack extends Stack { props.auditLogGroup.grantWrite(this.service.service.taskDefinition.taskRole) + // Ref: https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AWSXrayWriteOnlyAccess.html + this.service.taskDefinition.taskRole.addManagedPolicy( + ManagedPolicy.fromAwsManagedPolicyName("AWSXrayWriteOnlyAccess"), + ) + this.service.service .metricCpuUtilization() .createAlarm(this, "CpuUtilization", { diff --git a/otel-config.yml b/otel-config.yml new file mode 100644 index 00000000..cd24d4dd --- /dev/null +++ b/otel-config.yml @@ -0,0 +1,34 @@ +receivers: + otlp: + protocols: + grpc: + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +extensions: + health_check: + +exporters: + debug: + otlphttp/jaeger: + endpoint: http://jaeger:4318 + otlp/oteltui: + endpoint: oteltui:4317 + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug, otlphttp/jaeger, otlp/oteltui] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/oteltui] + logs: + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/oteltui] diff --git a/scripts/start_local_server.sh b/scripts/start_local_server.sh index 76a03d33..c566748d 100755 --- a/scripts/start_local_server.sh +++ b/scripts/start_local_server.sh @@ -8,10 +8,8 @@ get_secret() { aws secretsmanager get-secret-value --secret-id "$1" --output text --query SecretString } -require_command humanlog - require_env SPRING_PROFILES_ACTIVE require_env KOTLIN_POST_PROCESS_FILE cd "$REPO_ROOT"/server -./mvnw spring-boot:run | humanlog --truncate-length 9999 +./mvnw spring-boot:run diff --git a/server/pom.xml b/server/pom.xml index 05e6a2fd..ed8d0ce9 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -50,6 +50,13 @@ pom import + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-bom + 2.10.0 + pom + import + @@ -180,6 +187,14 @@ db-scheduler-log-spring-boot-starter 0.7.0 + + io.opentelemetry.instrumentation + opentelemetry-spring-boot-starter + + + org.springframework.boot + spring-boot-starter-aop + io.opentelemetry opentelemetry-api @@ -194,6 +209,11 @@ opentelemetry-semconv-incubating ${opentelemetry-semconv.version} + + io.opentelemetry.instrumentation + opentelemetry-aws-sdk-2.2-autoconfigure + 2.10.0-alpha + no.bekk.db-scheduler-ui db-scheduler-ui-starter diff --git a/server/src/main/kotlin/fi/oph/kitu/ErrorHandler.kt b/server/src/main/kotlin/fi/oph/kitu/ErrorHandler.kt index 39b7dfae..cf9e5422 100644 --- a/server/src/main/kotlin/fi/oph/kitu/ErrorHandler.kt +++ b/server/src/main/kotlin/fi/oph/kitu/ErrorHandler.kt @@ -1,7 +1,5 @@ package fi.oph.kitu -import org.slf4j.Logger -import org.slf4j.LoggerFactory import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.ControllerAdvice @@ -18,12 +16,9 @@ data class RestErrorMessage( @ControllerAdvice class GlobalControllerExceptionHandler { - private val logger: Logger = LoggerFactory.getLogger(GlobalControllerExceptionHandler::class.java) - @ExceptionHandler - fun handleRestClientException(ex: RestClientException): ResponseEntity { - logger.error(ex.stackTraceToString()) - return ResponseEntity( + fun handleRestClientException(ex: RestClientException): ResponseEntity = + ResponseEntity( RestErrorMessage( status = HttpStatus.SERVICE_UNAVAILABLE.value(), error = "Service Unavailable", @@ -31,5 +26,4 @@ class GlobalControllerExceptionHandler { ), HttpStatus.SERVICE_UNAVAILABLE, ) - } } diff --git a/server/src/main/kotlin/fi/oph/kitu/csvparsing/CsvParser.kt b/server/src/main/kotlin/fi/oph/kitu/csvparsing/CsvParser.kt index aedc1437..99eb966c 100644 --- a/server/src/main/kotlin/fi/oph/kitu/csvparsing/CsvParser.kt +++ b/server/src/main/kotlin/fi/oph/kitu/csvparsing/CsvParser.kt @@ -7,38 +7,25 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper import com.fasterxml.jackson.dataformat.csv.CsvSchema import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import fi.oph.kitu.logging.add +import io.opentelemetry.api.trace.Span import org.ietf.jgss.Oid -import org.slf4j.spi.LoggingEventBuilder import java.io.ByteArrayOutputStream import java.lang.RuntimeException import kotlin.reflect.full.findAnnotation class CsvParser( - val event: LoggingEventBuilder, val columnSeparator: Char = ',', val lineSeparator: String = "\n", val useHeader: Boolean = false, val quoteChar: Char = '"', ) { - init { - event.add( - "serialization.schema.args.columnSeparator" to columnSeparator.toString(), - "serialization.schema.args.lineSeparator" to lineSeparator, - "serialization.schema.args.useHeader" to useHeader, - "serialization.schema.args.quoteChar" to quoteChar, - ) - } - - inline fun getSchema(csvMapper: CsvMapper): CsvSchema { - event.add("serialization.schema.args.type" to T::class.java.name) - - return csvMapper + inline fun getSchema(csvMapper: CsvMapper): CsvSchema = + csvMapper .typedSchemaFor(T::class.java) .withColumnSeparator(columnSeparator) .withLineSeparator(lineSeparator) .withUseHeader(useHeader) .withQuoteChar(quoteChar) - } inline fun CsvMapper.Builder.withFeatures(): CsvMapper.Builder { val mapperFeatures = T::class.findAnnotation()?.features @@ -85,12 +72,9 @@ class CsvParser( */ inline fun convertCsvToData(csvString: String): List { if (csvString.isBlank()) { - event.add("serialization.isEmptyList" to true) return emptyList() } - event.add("serialization.isEmptyList" to false) - val csvMapper = getCsvMapper() val schema = getSchema(csvMapper) @@ -115,11 +99,13 @@ class CsvParser( return data } + val span = Span.current() + // add all errors to log errors.forEachIndexed { i, error -> - event.add("serialization.error[$i].index" to i) + span.setAttribute("serialization.error[$i].index", i.toLong()) for (kvp in error.keyValues) { - event.add("serialization.error[$i].${kvp.first}" to kvp.second) + span.setAttribute("serialization.error[$i].${kvp.first}", kvp.second.toString()) } } diff --git a/server/src/main/kotlin/fi/oph/kitu/kotoutumiskoulutus/KoealustaScheduledTasks.kt b/server/src/main/kotlin/fi/oph/kitu/kotoutumiskoulutus/KoealustaScheduledTasks.kt index 4642bf81..3cec3c6b 100644 --- a/server/src/main/kotlin/fi/oph/kitu/kotoutumiskoulutus/KoealustaScheduledTasks.kt +++ b/server/src/main/kotlin/fi/oph/kitu/kotoutumiskoulutus/KoealustaScheduledTasks.kt @@ -20,5 +20,7 @@ class KoealustaScheduledTasks { Tasks .recurring("Koto-import", Schedules.parseSchedule(koealustaImportSchedule), Instant::class.java) .initialData(Instant.EPOCH) - .executeStateful { taskInstance, _ -> koealustaService.importSuoritukset(taskInstance.data) } + .executeStateful { taskInstance, _ -> + koealustaService.importSuoritukset(taskInstance.data) + } } diff --git a/server/src/main/kotlin/fi/oph/kitu/kotoutumiskoulutus/KoealustaService.kt b/server/src/main/kotlin/fi/oph/kitu/kotoutumiskoulutus/KoealustaService.kt index 1fb273a0..eb922f7e 100644 --- a/server/src/main/kotlin/fi/oph/kitu/kotoutumiskoulutus/KoealustaService.kt +++ b/server/src/main/kotlin/fi/oph/kitu/kotoutumiskoulutus/KoealustaService.kt @@ -6,10 +6,7 @@ import com.fasterxml.jackson.module.kotlin.readValue import fi.oph.kitu.PeerService import fi.oph.kitu.logging.Logging import fi.oph.kitu.logging.add -import fi.oph.kitu.logging.addHttpResponse -import fi.oph.kitu.logging.withEventAndPerformanceCheck -import fi.oph.kitu.oppijanumero.addValidationExceptions -import org.slf4j.LoggerFactory +import io.opentelemetry.instrumentation.annotations.WithSpan import org.springframework.beans.factory.annotation.Value import org.springframework.http.MediaType import org.springframework.stereotype.Service @@ -24,7 +21,6 @@ class KoealustaService( private val jacksonObjectMapper: ObjectMapper, private val mappingService: KoealustaMappingService, ) { - private val logger = LoggerFactory.getLogger(javaClass) private val auditLogger = Logging.auditLogger() @Value("\${kitu.kotoutumiskoulutus.koealusta.wstoken}") @@ -57,62 +53,44 @@ class KoealustaService( } } - fun importSuoritukset(from: Instant) = - logger - .atInfo() - .withEventAndPerformanceCheck { event -> - event.add("from" to from) - - val response = - restClient - .get() - .uri( - "/webservice/rest/server.php?wstoken={token}&wsfunction={function}&moodlewsrestformat=json&from={from}", - mapOf( - "token" to koealustaToken, - "function" to "local_completion_export_get_completions", - "from" to from.epochSecond, - ), - ).accept(MediaType.APPLICATION_JSON) - .retrieve() - .toEntity() - - event - .add("request.token" to koealustaToken) - .addHttpResponse(PeerService.Koealusta, uri = "/webservice/rest/server.php", response) - - if (response.body == null) { - return@withEventAndPerformanceCheck from - } - - val suorituksetResponse = - tryParseMoodleResponse(response.body!!) + @WithSpan + fun importSuoritukset(from: Instant): Instant { + val response = + restClient + .get() + .uri( + "/webservice/rest/server.php?wstoken={token}&wsfunction={function}&moodlewsrestformat=json&from={from}", + mapOf( + "token" to koealustaToken, + "function" to "local_completion_export_get_completions", + "from" to from.epochSecond, + ), + ).accept(MediaType.APPLICATION_JSON) + .retrieve() + .toEntity() + + if (response.body == null) { + return from + } - val suoritukset = - try { - mappingService.convertToEntity(suorituksetResponse) - } catch (ex: KoealustaMappingService.Error.ValidationFailure) { - event.addValidationExceptions(ex.oppijanumeroExceptions, ex.validationErrors) - throw ex - } + val suorituksetResponse = + tryParseMoodleResponse(response.body!!) - val savedSuoritukset = kielitestiSuoritusRepository.saveAll(suoritukset) + val suoritukset = + mappingService.convertToEntity(suorituksetResponse) - event.add("db.saved" to savedSuoritukset.count()) + val savedSuoritukset = kielitestiSuoritusRepository.saveAll(suoritukset) - for (suoritus in savedSuoritukset) { - auditLogger - .atInfo() - .add( - "principal" to "koealusta.import", - "peer.service" to PeerService.Koealusta.value, - "suoritus.id" to suoritus.id, - ).log("Kielitesti suoritus imported") - } + for (suoritus in savedSuoritukset) { + auditLogger + .atInfo() + .add( + "principal" to "koealusta.import", + "peer.service" to PeerService.Koealusta.value, + "suoritus.id" to suoritus.id, + ).log("Kielitesti suoritus imported") + } - return@withEventAndPerformanceCheck suoritukset.maxOfOrNull { it.timeCompleted } ?: from - }.apply { - addDefaults("koealusta.importSuoritukset") - addDatabaseLogs() - }.getOrThrow() + return suoritukset.maxOfOrNull { it.timeCompleted } ?: from + } } diff --git a/server/src/main/kotlin/fi/oph/kitu/logging/AccessLoggingConfiguration.kt b/server/src/main/kotlin/fi/oph/kitu/logging/AccessLoggingConfiguration.kt deleted file mode 100644 index a0570372..00000000 --- a/server/src/main/kotlin/fi/oph/kitu/logging/AccessLoggingConfiguration.kt +++ /dev/null @@ -1,34 +0,0 @@ -package fi.oph.kitu.logging - -import jakarta.servlet.FilterChain -import jakarta.servlet.http.HttpServletRequest -import jakarta.servlet.http.HttpServletResponse -import org.slf4j.LoggerFactory -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.web.filter.OncePerRequestFilter - -@Configuration -class AccessLoggingConfiguration { - @Bean - fun logFilter(): OncePerRequestFilter = - object : OncePerRequestFilter() { - private val logger = LoggerFactory.getLogger(javaClass) - - override fun doFilterInternal( - request: HttpServletRequest, - response: HttpServletResponse, - filterChain: FilterChain, - ) { - try { - filterChain.doFilter(request, response) - } finally { - logger - .atInfo() - .addServletResponse(response) - .addServletRequest(request) - .log("HTTP ${request.method} ${request.requestURL}") - } - } - } -} diff --git a/server/src/main/kotlin/fi/oph/kitu/logging/CloudWatchAppender.kt b/server/src/main/kotlin/fi/oph/kitu/logging/CloudWatchAppender.kt index 69beef54..ccd8dbaa 100644 --- a/server/src/main/kotlin/fi/oph/kitu/logging/CloudWatchAppender.kt +++ b/server/src/main/kotlin/fi/oph/kitu/logging/CloudWatchAppender.kt @@ -3,7 +3,8 @@ package fi.oph.kitu.logging import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.core.AppenderBase import ch.qos.logback.core.encoder.Encoder -import org.slf4j.LoggerFactory +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest import software.amazon.awssdk.services.cloudwatchlogs.model.InputLogEvent @@ -11,8 +12,6 @@ import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsRequest // Copied from https://github.com/Opetushallitus/ludos/blob/main/server/src/main/kotlin/fi/oph/ludos/aws/LudosLogbackCloudwatchAppender.kt class CloudwatchAppender : AppenderBase() { - private val logger = LoggerFactory.getLogger(javaClass) - // var encoder: Encoder? = null var logGroupName: String? = null @@ -63,7 +62,12 @@ class CloudwatchAppender : AppenderBase() { try { cloudWatchLogsClient.putLogEvents(request) } catch (e: Throwable) { - logger.error("Error calling put-log-events(${eventObject.formattedMessage})", e) + // Don't propagate exception. + Span + .current() + .recordException( + e, + ).setStatus(StatusCode.ERROR, "Error calling put-log-events(${eventObject.formattedMessage})") } } } diff --git a/server/src/main/kotlin/fi/oph/kitu/oppijanumero/CasAuthenticatedService.kt b/server/src/main/kotlin/fi/oph/kitu/oppijanumero/CasAuthenticatedService.kt index c0de023d..a2632e65 100644 --- a/server/src/main/kotlin/fi/oph/kitu/oppijanumero/CasAuthenticatedService.kt +++ b/server/src/main/kotlin/fi/oph/kitu/oppijanumero/CasAuthenticatedService.kt @@ -1,8 +1,5 @@ package fi.oph.kitu.oppijanumero -import fi.oph.kitu.PeerService -import fi.oph.kitu.logging.addHttpResponse -import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Service @@ -20,8 +17,6 @@ class CasAuthenticatedServiceImpl( private val httpClient: HttpClient, private val casService: CasService, ) : CasAuthenticatedService { - private val logger = LoggerFactory.getLogger(javaClass) - @Value("\${kitu.oppijanumero.callerid}") private lateinit var callerId: String @@ -37,33 +32,17 @@ class CasAuthenticatedServiceImpl( .header("Caller-Id", callerId) .header("CSRF", "CSRF") .header("Cookie", "CSRF=CSRF") - val request = requestBuilder.build() - val response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) - logger.atInfo().addHttpResponse(PeerService.Oppijanumero, request.uri().toString(), response).log() - + val response = httpClient.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) if (isLoginToCas(response)) { // Oppijanumerorekisteri ohjaa CAS kirjautumissivulle, jos autentikaatiota // ei ole tehty. Luodaan uusi CAS ticket ja yritetään uudelleen. authenticateToCas() // gets JSESSIONID Cookie and it will be used in the next request below - val authenticatedRequest = requestBuilder.build() - val authenticatedResponse = httpClient.send(authenticatedRequest, HttpResponse.BodyHandlers.ofString()) - logger - .atInfo() - .addHttpResponse(PeerService.Oppijanumero, authenticatedRequest.uri().toString(), authenticatedResponse) - .log() - - return Result.success(authenticatedResponse) + return Result.success(httpClient.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString())) } else if (response.statusCode() == 401) { // Oppijanumerorekisteri vastaa HTTP 401 kun sessio on vanhentunut. // HUOM! Oppijanumerorekisteri vastaa HTTP 401 myös jos käyttöoikeudet eivät riitä. authenticateToCas() // gets JSESSIONID Cookie and it will be used in the next request below - val authenticatedRequest = requestBuilder.build() - val authenticatedResponse = httpClient.send(authenticatedRequest, HttpResponse.BodyHandlers.ofString()) - logger - .atInfo() - .addHttpResponse(PeerService.Oppijanumero, authenticatedRequest.uri().toString(), authenticatedResponse) - .log() - return Result.success(authenticatedResponse) + return Result.success(httpClient.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString())) } // loput statuskoodit oletetaan johtuvan kutsuttuvasta rajapinnasta diff --git a/server/src/main/kotlin/fi/oph/kitu/oppijanumero/CasService.kt b/server/src/main/kotlin/fi/oph/kitu/oppijanumero/CasService.kt index 0b7fb51c..9005f7a5 100644 --- a/server/src/main/kotlin/fi/oph/kitu/oppijanumero/CasService.kt +++ b/server/src/main/kotlin/fi/oph/kitu/oppijanumero/CasService.kt @@ -1,8 +1,5 @@ package fi.oph.kitu.oppijanumero -import fi.oph.kitu.PeerService -import fi.oph.kitu.logging.addHttpResponse -import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Service import java.net.URI @@ -15,8 +12,6 @@ import java.net.http.HttpResponse class CasService( private val httpClient: HttpClient, ) { - private val logger = LoggerFactory.getLogger(javaClass) - @Value("\${kitu.oppijanumero.username}") private lateinit var onrUsername: String @@ -35,8 +30,7 @@ class CasService( .newBuilder(URI.create("$serviceUrl/j_spring_cas_security_check?ticket=$serviceTicket")) .method("GET", HttpRequest.BodyPublishers.noBody()) .build() - val authResponse = httpClient.send(authRequest, HttpResponse.BodyHandlers.ofString()) - logger.atInfo().addHttpResponse(PeerService.Oppijanumero, authRequest.uri().toString(), authResponse).log() + httpClient.send(authRequest, HttpResponse.BodyHandlers.ofString()) } fun getServiceTicket(ticketGrantingTicket: String): String { @@ -51,14 +45,12 @@ class CasService( .build() val response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) - logger.atInfo().addHttpResponse(PeerService.Cas, request.uri().toString(), response).log() if (response.statusCode() != 200) { throw CasException(response, "Ticket service did not respond with 200 status code.") } - val ticket = response.body() - return ticket + return response.body() } fun getGrantingTicket(): String { @@ -74,7 +66,6 @@ class CasService( // Step 3 - Get the response val response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) - logger.atInfo().addHttpResponse(PeerService.Cas, request.uri().toString(), response).log() val statusCode = response.statusCode() diff --git a/server/src/main/kotlin/fi/oph/kitu/oppijanumero/OppijanumeroService.kt b/server/src/main/kotlin/fi/oph/kitu/oppijanumero/OppijanumeroService.kt index 9d339fdb..450db0b3 100644 --- a/server/src/main/kotlin/fi/oph/kitu/oppijanumero/OppijanumeroService.kt +++ b/server/src/main/kotlin/fi/oph/kitu/oppijanumero/OppijanumeroService.kt @@ -1,10 +1,8 @@ package fi.oph.kitu.oppijanumero import com.fasterxml.jackson.databind.ObjectMapper -import fi.oph.kitu.logging.add -import fi.oph.kitu.logging.addCondition -import fi.oph.kitu.logging.withEventAndPerformanceCheck -import org.slf4j.LoggerFactory +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.trace.Span import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Service import java.net.URI @@ -19,74 +17,72 @@ interface OppijanumeroService { class OppijanumeroServiceImpl( private val casAuthenticatedService: CasAuthenticatedService, val objectMapper: ObjectMapper, + openTelemetry: OpenTelemetry, ) : OppijanumeroService { - private val logger = LoggerFactory.getLogger(javaClass) - @Value("\${kitu.oppijanumero.service.url}") lateinit var serviceUrl: String @Value("\${kitu.oppijanumero.service.use-mock-data}") var useMockData: Boolean = false - override fun getOppijanumero(oppija: Oppija): String = - logger - .atInfo() - .withEventAndPerformanceCheck { event -> - require(oppija.etunimet.isNotEmpty()) { "etunimet cannot be empty" } - require(oppija.hetu.isNotEmpty()) { "hetu cannot be empty" } - require(oppija.sukunimi.isNotEmpty()) { "sukunimi cannot be empty" } - require(oppija.kutsumanimi.isNotEmpty()) { "kutsumanimi cannot be empty" } - - if (event.addCondition(key = "request.hasOppijanumero", condition = oppija.oppijanumero != null)) { - return@withEventAndPerformanceCheck oppija.oppijanumero.toString() - } - - if (event.addCondition(key = "useMockData", condition = useMockData)) { - return@withEventAndPerformanceCheck "1.2.246.562.24.33342764709" - } - - val endpoint = "$serviceUrl/yleistunniste/hae" - val httpRequest = - HttpRequest - .newBuilder(URI.create(endpoint)) - .POST( - HttpRequest.BodyPublishers.ofString( - objectMapper.writeValueAsString(oppija.toYleistunnisteHaeRequest()), - ), - ).header("Content-Type", "application/json") - - // no need to log sendRequest, because there are request and response logging inside casAuthenticatedService. - val stringResponse = - casAuthenticatedService - .sendRequest(httpRequest) - .getOrLogAndThrowCasException(event) - - if (stringResponse.statusCode() == 404) { - throw OppijanumeroException.OppijaNotFoundException(oppija) - } else if (stringResponse.statusCode() != 200) { - throw OppijanumeroException( - oppija, - "Oppijanumero-service returned unexpected status code ${stringResponse.statusCode()}", - ) - } - - val body = tryConvertToOppijanumeroResponse(oppija, stringResponse) - event.add( - "response.hasOppijanumero" to body.oppijanumero.isNullOrEmpty(), - "response.hasOid" to body.oid.isEmpty(), - "response.areOppijanumeroAndOidSame" to (body.oppijanumero == body.oid), - ) - - if (body.oppijanumero.isNullOrEmpty()) { - throw OppijanumeroException.OppijaNotIdentifiedException( - oppija.withYleistunnisteHaeResponse(body), - ) - } - - return@withEventAndPerformanceCheck body.oppijanumero - }.apply { - addDefaults("getOppijanumero") - }.getOrThrow() + private val tracer = openTelemetry.getTracer("oppijanumero-service") + + fun Span.setAttributeWithCondition( + key: String, + value: Boolean, + ): Boolean { + setAttribute(key, value) + return value + } + + override fun getOppijanumero(oppija: Oppija): String { + val span = tracer.spanBuilder("getOppijanumero").startSpan() + + require(oppija.etunimet.isNotEmpty()) { "etunimet cannot be empty" } + require(oppija.hetu.isNotEmpty()) { "hetu cannot be empty" } + require(oppija.sukunimi.isNotEmpty()) { "sukunimi cannot be empty" } + require(oppija.kutsumanimi.isNotEmpty()) { "kutsumanimi cannot be empty" } + + if (span.setAttributeWithCondition("requestHasOppijanumero", oppija.oppijanumero != null)) { + return oppija.oppijanumero.toString() + } + + if (span.setAttributeWithCondition("useMockData", useMockData)) { + return "1.2.246.562.24.33342764709" + } + + val endpoint = "$serviceUrl/yleistunniste/hae" + val httpRequest = + HttpRequest + .newBuilder(URI.create(endpoint)) + .POST( + HttpRequest.BodyPublishers.ofString( + objectMapper.writeValueAsString(oppija.toYleistunnisteHaeRequest()), + ), + ).header("Content-Type", "application/json") + + // no need to log sendRequest, because there are request and response logging inside casAuthenticatedService. + val stringResponse = casAuthenticatedService.sendRequest(httpRequest).getOrThrow() + + if (stringResponse.statusCode() == 404) { + throw OppijanumeroException.OppijaNotFoundException(oppija) + } else if (stringResponse.statusCode() != 200) { + throw OppijanumeroException( + oppija, + "Oppijanumero-service returned unexpected status code ${stringResponse.statusCode()}", + ) + } + + val body = tryConvertToOppijanumeroResponse(oppija, stringResponse) + + if (body.oppijanumero.isNullOrEmpty()) { + throw OppijanumeroException.OppijaNotIdentifiedException( + oppija.withYleistunnisteHaeResponse(body), + ) + } + + return body.oppijanumero + } /** * Tries to convert HttpResponse into the given T. diff --git a/server/src/main/kotlin/fi/oph/kitu/yki/YkiScheduledTasks.kt b/server/src/main/kotlin/fi/oph/kitu/yki/YkiScheduledTasks.kt index ce19e9e4..a9ecde1a 100644 --- a/server/src/main/kotlin/fi/oph/kitu/yki/YkiScheduledTasks.kt +++ b/server/src/main/kotlin/fi/oph/kitu/yki/YkiScheduledTasks.kt @@ -23,7 +23,9 @@ class YkiScheduledTasks { Tasks .recurring("YKI-import", Schedules.parseSchedule(ykiImportSchedule), Instant::class.java) .initialData(null) - .execute { taskInstance, _ -> ykiService.importYkiSuoritukset(taskInstance.data) } + .execute { taskInstance, _ -> + ykiService.importYkiSuoritukset(taskInstance.data) + } @Bean fun arvioijatImport(ykiService: YkiService): Task = diff --git a/server/src/main/kotlin/fi/oph/kitu/yki/YkiService.kt b/server/src/main/kotlin/fi/oph/kitu/yki/YkiService.kt index 1b582136..2f5b4373 100644 --- a/server/src/main/kotlin/fi/oph/kitu/yki/YkiService.kt +++ b/server/src/main/kotlin/fi/oph/kitu/yki/YkiService.kt @@ -4,9 +4,7 @@ import fi.oph.kitu.PeerService import fi.oph.kitu.csvparsing.CsvParser import fi.oph.kitu.logging.Logging import fi.oph.kitu.logging.add -import fi.oph.kitu.logging.addHttpResponse import fi.oph.kitu.logging.addUser -import fi.oph.kitu.logging.withEventAndPerformanceCheck import fi.oph.kitu.yki.arvioijat.SolkiArvioijaResponse import fi.oph.kitu.yki.arvioijat.YkiArvioijaEntity import fi.oph.kitu.yki.arvioijat.YkiArvioijaMappingService @@ -15,15 +13,15 @@ import fi.oph.kitu.yki.suoritukset.YkiSuoritusCsv import fi.oph.kitu.yki.suoritukset.YkiSuoritusEntity import fi.oph.kitu.yki.suoritukset.YkiSuoritusMappingService import fi.oph.kitu.yki.suoritukset.YkiSuoritusRepository +import io.opentelemetry.api.trace.Span +import io.opentelemetry.instrumentation.annotations.WithSpan import org.slf4j.Logger -import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Qualifier import org.springframework.stereotype.Service import org.springframework.web.client.RestClient import org.springframework.web.client.toEntity import java.io.ByteArrayOutputStream import java.time.Instant -import java.time.LocalDate import java.time.format.DateTimeFormatter @Service @@ -35,122 +33,98 @@ class YkiService( private val arvioijaRepository: YkiArvioijaRepository, private val arvioijaMapper: YkiArvioijaMappingService, ) { - private val logger: Logger = LoggerFactory.getLogger(javaClass) private val auditLogger: Logger = Logging.auditLogger() + @WithSpan fun importYkiSuoritukset( from: Instant? = null, - lastSeen: LocalDate? = null, dryRun: Boolean? = null, - ): Instant? = - logger - .atInfo() - .withEventAndPerformanceCheck { event -> - val parser = CsvParser(event) - event.add("dryRun" to dryRun, "lastSeen" to lastSeen) - - val url = - if (from != null) { - "suoritukset?m=${DateTimeFormatter.ISO_INSTANT.format(from)}" - } else { - "suoritukset" - } - val response = - solkiRestClient - .get() - .uri(url) - .retrieve() - .toEntity() - - event.addHttpResponse(PeerService.Solki, "suoritukset", response) - - val suoritukset = - parser - .convertCsvToData(response.body ?: "") - .also { - for (suoritus in it) { - auditLogger - .atInfo() - .add( - "principal" to "yki.importSuoritukset", - "suoritus.id" to suoritus.suoritusID, - ) - } - } - - if (dryRun != true) { - val res = suoritusRepository.saveAll(suoritusMapper.convertToEntityIterable(suoritukset)) - event.add("importedSuorituksetSize" to res.count()) - } - return@withEventAndPerformanceCheck suoritukset.maxOfOrNull { it.lastModified } ?: from - }.apply { - addDefaults("yki.importSuoritukset") - addDatabaseLogs() - }.getOrThrow() - - fun importYkiArvioijat(dryRun: Boolean = false) = - logger - .atInfo() - .withEventAndPerformanceCheck { event -> - val parser = CsvParser(event) - val response = - solkiRestClient - .get() - .uri("arvioijat") - .retrieve() - .toEntity() - - event.addHttpResponse(PeerService.Solki, "arvioijat", response) - - val arvioijat = - parser.convertCsvToData( - response.body ?: throw Error.EmptyArvioijatResponse(), - ) - - event.add("yki.arvioijat.receivedCount" to arvioijat.size) - - if (arvioijat.isEmpty()) { - throw Error.EmptyArvioijat() - } - - if (!dryRun) { - val importedArvioijat = - arvioijaRepository.saveAll( - arvioijaMapper.convertToEntityIterable(arvioijat), - ) - event.add("yki.arvioijat.importedCount" to importedArvioijat.count()) - - for (arvioija in importedArvioijat) { + ): Instant? { + val parser = CsvParser() + + val span = Span.current() + + val url = if (from != null) "suoritukset?m=${DateTimeFormatter.ISO_INSTANT.format(from)}" else "suoritukset" + span.setAttribute("url", url) + + val response = + solkiRestClient + .get() + .uri(url) + .retrieve() + .toEntity() + + val suoritukset = + parser + .convertCsvToData(response.body ?: "") + .also { + for (suoritus in it) { auditLogger .atInfo() .add( - "principal" to "yki.importArvioijat", - "peer.service" to PeerService.Solki.value, - "arvioija.oppijanumero" to arvioija.arvioijanOppijanumero, - ).log("YKI arvioija imported") + "principal" to "yki.importSuoritukset", + "suoritus.id" to suoritus.suoritusID, + ) } } - }.apply { - addDefaults("yki.importArvioijat") - addDatabaseLogs() - }.getOrThrow() - - fun generateSuorituksetCsvStream(includeVersionHistory: Boolean): ByteArrayOutputStream = - logger - .atInfo() - .withEventAndPerformanceCheck { event -> - val parser = CsvParser(event, useHeader = true) - val suoritukset = allSuoritukset(includeVersionHistory) - event.add("dataCount" to suoritukset.count()) - val writableData = suoritusMapper.convertToResponseIterable(suoritukset) - val outputStream = ByteArrayOutputStream() - parser.streamDataAsCsv(outputStream, writableData) - - return@withEventAndPerformanceCheck outputStream - }.apply { - addDefaults("yki.getSuorituksetCsv") - addDatabaseLogs() - }.getOrThrow() + + if (dryRun != true) { + suoritusRepository.saveAll(suoritusMapper.convertToEntityIterable(suoritukset)) + } + + return suoritukset.maxOfOrNull { it.lastModified } ?: from + } + + @WithSpan + fun importYkiArvioijat(dryRun: Boolean = false) { + val parser = CsvParser() + + val span = Span.current() + + val response = + solkiRestClient + .get() + .uri("arvioijat") + .retrieve() + .toEntity() + + val arvioijat = + parser.convertCsvToData( + response.body ?: throw Error.EmptyArvioijatResponse(), + ) + + span.setAttribute("yki.arvioijat.receivedCount", arvioijat.size.toLong()) + + if (arvioijat.isEmpty()) { + throw Error.EmptyArvioijat() + } + + if (dryRun) { + return + } + + val importedArvioijat = arvioijaRepository.saveAll(arvioijaMapper.convertToEntityIterable(arvioijat)) + + for (arvioija in importedArvioijat) { + auditLogger + .atInfo() + .add( + "principal" to "yki.importArvioijat", + "peer.service" to PeerService.Solki.value, + "arvioija.oppijanumero" to arvioija.arvioijanOppijanumero, + ).log("YKI arvioija imported") + } + } + + fun generateSuorituksetCsvStream(includeVersionHistory: Boolean): ByteArrayOutputStream { + val parser = CsvParser(useHeader = true) + val suoritukset = allSuoritukset(includeVersionHistory) + Span.current().setAttribute("suoritus.count", suoritukset.count().toLong()) + val writableData = suoritusMapper.convertToResponseIterable(suoritukset) + val outputStream = ByteArrayOutputStream() + parser.streamDataAsCsv(outputStream, writableData) + return outputStream + } fun allSuoritukset(versionHistory: Boolean?): List = if (versionHistory == true) { diff --git a/server/src/main/kotlin/fi/oph/kitu/yki/YkiViewController.kt b/server/src/main/kotlin/fi/oph/kitu/yki/YkiViewController.kt index 03cdbb36..d60b37e3 100644 --- a/server/src/main/kotlin/fi/oph/kitu/yki/YkiViewController.kt +++ b/server/src/main/kotlin/fi/oph/kitu/yki/YkiViewController.kt @@ -1,5 +1,6 @@ package fi.oph.kitu.yki +import io.opentelemetry.instrumentation.annotations.WithSpan import org.springframework.stereotype.Controller import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestMapping @@ -12,6 +13,7 @@ class YkiViewController( private val ykiService: YkiService, ) { @GetMapping("/suoritukset", produces = ["text/html"]) + @WithSpan fun suorituksetView( @RequestParam("versionHistory") versionHistory: Boolean?, ): ModelAndView = @@ -20,6 +22,7 @@ class YkiViewController( .addObject("versionHistory", versionHistory == true) @GetMapping("/arvioijat") + @WithSpan fun arvioijatView(): ModelAndView = ModelAndView("yki-arvioijat") .addObject("arvioijat", ykiService.allArvioijat()) diff --git a/server/src/main/kotlin/fi/oph/kitu/yki/arvioijat/YkiArvioijaRepository.kt b/server/src/main/kotlin/fi/oph/kitu/yki/arvioijat/YkiArvioijaRepository.kt index a362c8cb..8633bed9 100644 --- a/server/src/main/kotlin/fi/oph/kitu/yki/arvioijat/YkiArvioijaRepository.kt +++ b/server/src/main/kotlin/fi/oph/kitu/yki/arvioijat/YkiArvioijaRepository.kt @@ -2,6 +2,7 @@ package fi.oph.kitu.yki.arvioijat import fi.oph.kitu.yki.Tutkintotaso import fi.oph.kitu.yki.getTutkintokieli +import io.opentelemetry.instrumentation.annotations.WithSpan import org.springframework.beans.factory.annotation.Autowired import org.springframework.data.repository.CrudRepository import org.springframework.jdbc.core.JdbcTemplate @@ -23,6 +24,7 @@ class CustomYkiArvioijaRepositoryImpl : CustomYkiArvioijaRepository { * Override to allow handling duplicates/conflicts. The default implementation from CrudRepository fails * due to the unique constraint. Overriding the implementation allows explicit handling of conflicts. */ + @WithSpan override fun saveAll(arvioijat: Iterable): Iterable { val sql = """ diff --git a/server/src/main/resources/application-prod.properties b/server/src/main/resources/application-prod.properties index 61c12db5..4ad36e76 100644 --- a/server/src/main/resources/application-prod.properties +++ b/server/src/main/resources/application-prod.properties @@ -17,3 +17,5 @@ kitu.yki.scheduling.enabled=true kitu.yki.baseUrl=https://yki.jyu.fi/yki-sp/oph/ kitu.yki.scheduling.import.schedule=DAILY|03:00 kitu.yki.scheduling.importArvioijat.schedule=DAILY|03:00 + +otel.resource.providers.aws.enabled=true diff --git a/server/src/main/resources/application-qa.properties b/server/src/main/resources/application-qa.properties index 13cd5929..91e42361 100644 --- a/server/src/main/resources/application-qa.properties +++ b/server/src/main/resources/application-qa.properties @@ -16,3 +16,5 @@ kitu.yki.scheduling.enabled=true kitu.yki.scheduling.import.schedule=DAILY|03:00 kitu.yki.scheduling.importArvioijat.schedule=DAILY|03:00 kitu.yki.baseUrl=https://yki-test.cc.jyu.fi/yki-sp/oph/ + +otel.resource.providers.aws.enabled=true diff --git a/server/src/main/resources/application-untuva.properties b/server/src/main/resources/application-untuva.properties index d2a842f8..bc42962e 100644 --- a/server/src/main/resources/application-untuva.properties +++ b/server/src/main/resources/application-untuva.properties @@ -16,3 +16,5 @@ kitu.yki.scheduling.enabled=true kitu.yki.scheduling.import.schedule=DAILY|03:00 kitu.yki.scheduling.importArvioijat.schedule=DAILY|03:00 kitu.yki.baseUrl=https://yki-test.cc.jyu.fi/yki-sp/oph/ + +otel.resource.providers.aws.enabled=true diff --git a/server/src/main/resources/application.properties b/server/src/main/resources/application.properties index a1647740..4e0ce2d5 100644 --- a/server/src/main/resources/application.properties +++ b/server/src/main/resources/application.properties @@ -29,7 +29,13 @@ kitu.yki.password=${YKI_API_PASSWORD} spring.mustache.suffix=.mustache -logging.structured.format.console=ecs +# Disable Spring console logger, use OTEL logger instead. +logging.pattern.console= +otel.logs.exporter=otlp,console + +# Enable OTEL traces, render human-readable versions to console and send full traces to default endpoint (see docker-compose.yml). +otel.traces.exporter=otlp,console + logging.structured.format.file=ecs # Uncomment this to debug authentication/authorization issues. diff --git a/server/src/main/resources/logback-spring.xml b/server/src/main/resources/logback-spring.xml index d4460dab..66f5d043 100644 --- a/server/src/main/resources/logback-spring.xml +++ b/server/src/main/resources/logback-spring.xml @@ -1,7 +1,6 @@ - logs/audit.log @@ -19,10 +18,6 @@ - - - - diff --git a/server/src/test/kotlin/fi/oph/kitu/csvparsing/CsvParsingTest.kt b/server/src/test/kotlin/fi/oph/kitu/csvparsing/CsvParsingTest.kt index 413fcc6d..12524028 100644 --- a/server/src/test/kotlin/fi/oph/kitu/csvparsing/CsvParsingTest.kt +++ b/server/src/test/kotlin/fi/oph/kitu/csvparsing/CsvParsingTest.kt @@ -9,6 +9,7 @@ import fi.oph.kitu.yki.Tutkintotaso import fi.oph.kitu.yki.arvioijat.SolkiArvioijaResponse import fi.oph.kitu.yki.suoritukset.YkiSuoritusCsv import org.ietf.jgss.Oid +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import java.io.ByteArrayOutputStream @@ -23,7 +24,7 @@ import kotlin.test.assertTrue class CsvParsingTest { @Test fun `test yki suoritukset parsing`() { - val parser = CsvParser(MockEvent()) + val parser = CsvParser() val csv = """ "1.2.246.562.24.20281155246","010180-9026","N","Öhman-Testi","Ranja Testi","EST","Testikuja 5","40100","Testilä","testi@testi.fi",183424,2024-10-30T13:53:56Z,2024-09-01,"fin","YT","1.2.246.562.10.14893989377","Jyväskylän yliopisto, Soveltavan kielentutkimuksen keskus",2024-11-14,5,5,,5,5,,,,0,0,, @@ -66,7 +67,7 @@ class CsvParsingTest { @Test fun `test line breaks`() { - val parser = CsvParser(MockEvent()) + val parser = CsvParser() val perustelut1 = " - Hyvä kielioppi\n - Selkeä puhuminen\n - Ymmärtää hyvin puhetta\n" val perustelut2 = " - Hyvä kielioppi\r\n - Selkeä puhuminen\r\n - Ymmärtää hyvin puhetta\r\n" // you can't use trimIndent here, because the string contains CR (\r) @@ -84,7 +85,7 @@ class CsvParsingTest { @Test fun `test legacy language code 10 parsing`() { - val parser = CsvParser(MockEvent()) + val parser = CsvParser() val arvioijaCsv = """ "1.2.246.562.24.24941612410","010180-922U","Torvinen-Testi","Anniina Testi","anniina.testi@yki.fi","Testiosoite 7357","00100","HELSINKI",1994-08-01,2019-06-29,2024-06-29,0,0,"10","PT+KT" @@ -95,7 +96,7 @@ class CsvParsingTest { @Test fun `test legacy language code 11 parsing`() { - val parser = CsvParser(MockEvent()) + val parser = CsvParser() val arvioijaCsv = """ "1.2.246.562.24.24941612410","010180-922U","Torvinen-Testi","Anniina Testi","anniina.testi@yki.fi","Testiosoite 7357","00100","HELSINKI",1994-08-01,2019-06-29,2024-06-29,0,0,"11","PT+KT" @@ -106,7 +107,7 @@ class CsvParsingTest { @Test fun `test legacy language code 12 parsing`() { - val parser = CsvParser(MockEvent()) + val parser = CsvParser() val arvioijaCsv = """ "1.2.246.562.24.24941612410","010180-922U","Torvinen-Testi","Anniina Testi","anniina.testi@yki.fi","Testiosoite 7357","00100","HELSINKI",1994-08-01,2019-06-29,2024-06-29,0,0,"12","PT+KT" @@ -117,7 +118,7 @@ class CsvParsingTest { @Test fun `test parsing yki suoritus with newlines`() { - val parser = CsvParser(MockEvent()) + val parser = CsvParser() val csv = """ "1.2.246.562.24.20281155246","010180-9026","N","Öhman-Testi","Ranja Testi","EST","Testikuja 5","40100","Testilä","testi@testi.fi",183424,2024-10-30T13:53:56Z,2024-09-01,"fin","YT","1.2.246.562.10.14893989377","Jyväskylän yliopisto, Soveltavan kielentutkimuksen keskus",2024-11-14,5,5,,5,5,,,,0,0,"Tarkistusarvioinnin perustelu\nJossa rivinvaihto", @@ -159,9 +160,10 @@ class CsvParsingTest { } @Test + @Disabled("Needs re-implementing with OpenTelemetry") fun `parsing errors are logged `() { val event = MockEvent() - val parser = CsvParser(event) + val parser = CsvParser() val csv = """ "INVALID_OID","010180-9026","N","Öhman-Testi","Ranja Testi","EST","Testikuja 5","40100","Testilä","testi@testi.fi",183424,2024-10-30T13:53:56Z,2024-09-01,"fin","YT","1.2.246.562.10.14893989377","Jyväskylän yliopisto, Soveltavan kielentutkimuksen keskus",2024-11-14,5,5,,5,5,,,,0,0,, @@ -200,7 +202,7 @@ class CsvParsingTest { fun `test writing csv`() { val datePattern = "yyyy-MM-dd" val dateFormatter = DateTimeFormatter.ofPattern(datePattern) - val parser = CsvParser(MockEvent(), useHeader = true) + val parser = CsvParser(useHeader = true) val writable = listOf( YkiSuoritusCsv( @@ -251,7 +253,7 @@ class CsvParsingTest { fun `null values are written correctly to csv`() { val datePattern = "yyyy-MM-dd" val dateFormatter = DateTimeFormatter.ofPattern(datePattern) - val parser = CsvParser(MockEvent(), useHeader = true) + val parser = CsvParser(useHeader = true) val writable = listOf( YkiSuoritusCsv( diff --git a/server/src/test/kotlin/fi/oph/kitu/oppijanumero/OppijanumeroServiceTests.kt b/server/src/test/kotlin/fi/oph/kitu/oppijanumero/OppijanumeroServiceTests.kt index 7c266c05..b7b73daa 100644 --- a/server/src/test/kotlin/fi/oph/kitu/oppijanumero/OppijanumeroServiceTests.kt +++ b/server/src/test/kotlin/fi/oph/kitu/oppijanumero/OppijanumeroServiceTests.kt @@ -2,10 +2,13 @@ package fi.oph.kitu.oppijanumero import HttpResponseMock import com.fasterxml.jackson.databind.ObjectMapper +import io.opentelemetry.api.OpenTelemetry import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows class OppijanumeroServiceTests { + val openTelemetry: OpenTelemetry = OpenTelemetry.noop() + @Test fun `oppijanumero service returns identified user`() { // Facade @@ -28,6 +31,7 @@ class OppijanumeroServiceTests { casAuthenticatedService = CasAuthenticatedServiceMock(response), objectMapper = ObjectMapper(), + openTelemetry = openTelemetry, ) oppijanumeroService.serviceUrl = "http://localhost:8080/oppijanumero-service" @@ -63,6 +67,7 @@ class OppijanumeroServiceTests { casAuthenticatedService = CasAuthenticatedServiceMock(response), objectMapper = ObjectMapper(), + openTelemetry = openTelemetry, ) oppijanumeroService.serviceUrl = "http://localhost:8080/oppijanumero-service" @@ -102,6 +107,7 @@ class OppijanumeroServiceTests { casAuthenticatedService = CasAuthenticatedServiceMock(response), objectMapper = ObjectMapper(), + openTelemetry = openTelemetry, ) oppijanumeroService.serviceUrl = "http://localhost:8080/oppijanumero-service" diff --git a/server/src/test/kotlin/fi/oph/kitu/yki/YkiServiceTests.kt b/server/src/test/kotlin/fi/oph/kitu/yki/YkiServiceTests.kt index 1a5c8df1..bb99f553 100644 --- a/server/src/test/kotlin/fi/oph/kitu/yki/YkiServiceTests.kt +++ b/server/src/test/kotlin/fi/oph/kitu/yki/YkiServiceTests.kt @@ -74,7 +74,7 @@ class YkiServiceTests( ) // Act - ykiService.importYkiSuoritukset(null, null, false) + ykiService.importYkiSuoritukset(null, false) // Assert val suoritukset = ykiSuoritusRepository.findAll() @@ -109,7 +109,7 @@ class YkiServiceTests( // Act assertThrows { - ykiService.importYkiSuoritukset(null, null, false) + ykiService.importYkiSuoritukset(null, false) } val suoritukset = ykiSuoritusRepository.findAll() assertEquals(0, suoritukset.count()) @@ -144,7 +144,7 @@ class YkiServiceTests( ) // Act - val from = ykiService.importYkiSuoritukset(null, null, false) + val from = ykiService.importYkiSuoritukset(null, false) // Assert val firstSuoritukset = ykiSuoritusRepository.findAll() @@ -164,7 +164,7 @@ class YkiServiceTests( ) // Act - ykiService.importYkiSuoritukset(from, null, false) + ykiService.importYkiSuoritukset(from, false) // Assert val suoritukset = ykiSuoritusRepository.findAll() diff --git a/server/src/test/resources/application.properties b/server/src/test/resources/application.properties index e80239cf..4bd5d6c1 100644 --- a/server/src/test/resources/application.properties +++ b/server/src/test/resources/application.properties @@ -21,5 +21,12 @@ kitu.yki.baseUrl= kitu.yki.username= kitu.yki.password= -logging.structured.format.console=ecs +# Disable Spring console logger, use OTEL logger instead. +logging.pattern.console= +otel.logs.exporter=otlp,console + +# Enable OTEL traces, render human-readable versions to console and send full traces to default endpoint (see docker-compose.yml). +otel.traces.exporter=otlp,console + logging.structured.format.file=ecs +