Moving download report over from being A CLI command to being an API

prime-router/docs/api/reports.yml

@@ -102,7 +102,36 @@ paths:
                 $ref: '#/components/schemas/Report'
         '500':
           description: Internal Server Error
-
+  /reports/download:
+    get:
+      summary: Downloads a message based on the report id
+      security:
+        - OAuth2: [ system_admin ]
+      parameters:
+        - in: query
+          name: reportId
+          description: The report id to look for to download.
+          schema:
+            type: string
+          required: true
+          example: e491f4fb-f2c5-4473-8db2-206ea04991e8
+        - in: query
+          name: removePII
+          description: Boolean that determines if PII will be removed from the message. If missing will default to true.
+            Required to be true if prod env.
+          required: false
+          schema:
+            type: boolean
+          example: true
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Report'
+        '500':
+          description: Internal Server Error
 # Building
 components:
   schemas:

prime-router/src/main/kotlin/azure/ReportFunction.kt

@@ -15,6 +15,7 @@ import gov.cdc.prime.router.ActionLogLevel
 import gov.cdc.prime.router.InvalidParamMessage
 import gov.cdc.prime.router.InvalidReportMessage
 import gov.cdc.prime.router.Options
+import gov.cdc.prime.router.ReportId
 import gov.cdc.prime.router.Sender
 import gov.cdc.prime.router.Sender.ProcessingType
 import gov.cdc.prime.router.SubmissionReceiver
@@ -24,13 +25,19 @@ import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService
+import gov.cdc.prime.router.cli.CommandUtilities.Companion.abort
+import gov.cdc.prime.router.cli.PIIRemovalCommands
 import gov.cdc.prime.router.common.AzureHttpUtils.getSenderIP
+import gov.cdc.prime.router.common.Environment
 import gov.cdc.prime.router.common.JacksonMapperUtilities
+import gov.cdc.prime.router.fhirengine.utils.FhirTranscoder
 import gov.cdc.prime.router.history.azure.SubmissionsFacade
 import gov.cdc.prime.router.tokens.AuthenticatedClaims
+import gov.cdc.prime.router.tokens.Scope
 import gov.cdc.prime.router.tokens.authenticationFailure
 import gov.cdc.prime.router.tokens.authorizationFailure
 import org.apache.logging.log4j.kotlin.Logging
+import java.util.UUID
 
 private const val PROCESSING_TYPE_PARAMETER = "processing"
 
@@ -85,6 +92,71 @@ class ReportFunction(
         }
     }
 
+    /**
+     * GET report to download
+     *
+     * @see ../../../docs/api/reports.yml
+     */
+    @FunctionName("getMessagesFromTestBank")
+    fun downloadReport(
+        @HttpTrigger(
+            name = "downloadReport",
+            methods = [HttpMethod.GET],
+            authLevel = AuthorizationLevel.ANONYMOUS,
+            route = "reports/download"
+        ) request: HttpRequestMessage<String?>,
+    ): HttpResponseMessage {
+        val claims = AuthenticatedClaims.authenticate(request)
+        if (claims != null && claims.authorized(setOf(Scope.primeAdminScope))) {
+            val reportId = request.queryParameters[REPORT_ID_PARAMETER]
+            val removePIIRaw = request.queryParameters[REMOVE_PII]
+            var removePII = false
+            if (removePIIRaw.isNullOrBlank() || removePIIRaw.toBoolean()) {
+                removePII = true
+            }
+            if (reportId.isNullOrBlank()) {
+                return HttpUtilities.badRequestResponse(request, "Must provide a reportId.")
+            }
+            return processDownloadReport(
+                request,
+                ReportId.fromString(reportId),
+                removePII
+            )
+        }
+        return HttpUtilities.unauthorizedResponse(request)
+    }
+
+    fun processDownloadReport(
+        request: HttpRequestMessage<String?>,
+        reportId: UUID,
+        removePII: Boolean?,
+        databaseAccess: DatabaseAccess = DatabaseAccess(),
+    ): HttpResponseMessage {
+        val requestedReport = databaseAccess.fetchReportFile(reportId)
+
+        return if (requestedReport.bodyUrl != null && requestedReport.bodyUrl.toString().lowercase().endsWith("fhir")) {
+            val contents = BlobAccess.downloadBlobAsByteArray(requestedReport.bodyUrl)
+
+            val content = if (removePII == null || removePII) {
+                PIIRemovalCommands().removePii(FhirTranscoder.decode(contents.toString(Charsets.UTF_8)))
+            } else {
+                if (Environment.get().envName == "prod") {
+                    abort("Must remove PII for messages from prod.")
+                }
+
+                val jsonObject = JacksonMapperUtilities.defaultMapper
+                    .readValue(contents.toString(Charsets.UTF_8), Any::class.java)
+                JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject)
+            }
+
+            HttpUtilities.okJSONResponse(request, content)
+        } else if (requestedReport.bodyUrl == null) {
+            HttpUtilities.badRequestResponse(request, "The requested report does not exist.")
+        } else {
+            HttpUtilities.badRequestResponse(request, "The requested report is not fhir.")
+        }
+    }
+
     /**
      * The Waters API, in memory of Dr. Michael Waters
      * (The older version of this API is "/api/reports")

prime-router/src/main/kotlin/azure/RequestFunction.kt

@@ -23,6 +23,8 @@ const val ALLOW_DUPLICATES_PARAMETER = "allowDuplicate"
 const val TOPIC_PARAMETER = "topic"
 const val SCHEMA_PARAMETER = "schema"
 const val FORMAT_PARAMETER = "format"
+const val REPORT_ID_PARAMETER = "reportId"
+const val REMOVE_PII = "removePII"
 
 /**
  * Base class for ReportFunction and ValidateFunction

prime-router/src/main/kotlin/cli/main.kt

@@ -300,7 +300,6 @@ fun main(args: Array<String>) = RouterCli()
         ValidateTranslationSchemaCommand(),
         SyncTranslationSchemaCommand(),
         ValidateYAMLCommand(),
-        PIIRemovalCommands(),
-        DownloadReport()
+        PIIRemovalCommands()
     ).context { terminal = Terminal(ansiLevel = AnsiLevel.TRUECOLOR) }
     .main(args)

prime-router/src/test/kotlin/azure/ReportFunctionTests.kt

@@ -24,6 +24,7 @@ import gov.cdc.prime.router.Topic
 import gov.cdc.prime.router.TopicReceiver
 import gov.cdc.prime.router.UniversalPipelineSender
 import gov.cdc.prime.router.azure.db.enums.TaskAction
+import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile
 import gov.cdc.prime.router.history.DetailedSubmissionHistory
 import gov.cdc.prime.router.history.azure.SubmissionsFacade
 import gov.cdc.prime.router.serializers.Hl7Serializer
@@ -45,6 +46,7 @@ import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import java.time.OffsetDateTime
 import java.util.UUID
+import kotlin.test.assertFailsWith
 
 class ReportFunctionTests {
     val dataProvider = MockDataProvider { emptyArray<MockResult>() }
@@ -172,6 +174,11 @@ class ReportFunctionTests {
         "05D2222542&ISO||445297001^Swab of internal nose^SCT^^^^2.67||||53342003^Internal nose structure" +
         " (body structure)^SCT^^^^2020-09-01|||||||||202108020000-0500|20210802000006.0000-0500"
 
+    @Suppress("ktlint:standard:max-line-length")
+    val fhirReport = """{"resourceType":"Bundle","id":"1667861767830636000.7db38d22-b713-49fc-abfa-2edba9c12347",
+        |"meta":{"lastUpdated":"2022-11-07T22:56:07.832+00:00"},"identifier":{"value":"1234d1d1-95fe-462c-8ac6-46728dba581c"},"type":"message","timestamp":"2021-08-03T13:15:11.015+00:00","entry":[{"fullUrl":"Observation/d683b42a-bf50-45e8-9fce-6c0531994f09","resource":{"resourceType":"Observation","id":"d683b42a-bf50-45e8-9fce-6c0531994f09","status":"final","code":{"coding":[{"system":"http://loinc.org","code":"80382-5"}],"text":"Flu A"},"subject":{"reference":"Patient/9473889b-b2b9-45ac-a8d8-191f27132912"},"performer":[{"reference":"Organization/1a0139b9-fc23-450b-9b6c-cd081e5cea9d"}],"valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"260373001","display":"Detected"}]},"interpretation":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0078","code":"A","display":"Abnormal"}]}],"method":{"extension":[{"url":"https://reportstream.cdc.gov/fhir/StructureDefinition/testkit-name-id","valueCoding":{"code":"BD Veritor System for Rapid Detection of SARS-CoV-2 & Flu A+B_Becton, Dickinson and Company (BD)"}},{"url":"https://reportstream.cdc.gov/fhir/StructureDefinition/equipment-uid","valueCoding":{"code":"BD Veritor System for Rapid Detection of SARS-CoV-2 & Flu A+B_Becton, Dickinson and Company (BD)"}}],"coding":[{"display":"BD Veritor System for Rapid Detection of SARS-CoV-2 & Flu A+B*"}]},"specimen":{"reference":"Specimen/52a582e4-d389-42d0-b738-bee51cf5244d"},"device":{"reference":"Device/78dc4d98-2958-43a3-a445-76ceef8c0698"}}}]}
+""".trimMargin()
+
     private fun makeEngine(metadata: Metadata, settings: SettingsProvider): WorkflowEngine = spyk(
         WorkflowEngine.Builder().metadata(metadata).settingsProvider(settings).databaseAccess(accessSpy)
             .blobAccess(blobMock).queueAccess(queueMock).hl7Serializer(serializer).build()
@@ -730,4 +737,115 @@ class ReportFunctionTests {
             reportFunc.extractPayloadName(mockHttpRequest)
         }
     }
+
+    @Test
+    fun `No report`() {
+        val mockDb = mockk<DatabaseAccess>()
+        val reportId = UUID.randomUUID()
+        val reportFunc = spyk(ReportFunction(engine, actionHistory))
+        every { mockDb.fetchReportFile(reportId, null, null) } returns ReportFile()
+//        every { mockDb.fetchReportFile(reportId, null, null) } returns error("Could not find $reportId in REPORT_FILE")
+        assertFailsWith<IllegalStateException>(
+            block = {
+                ReportFunction().processDownloadReport(MockHttpRequestMessage(), reportId, true, mockDb)
+            }
+        )
+    }
+//
+//    @Test
+//    fun `valid access token, report found, PII removal`() {
+//        val reportFile = ReportFile()
+//        reportFile.bodyUrl = "fakeurl.fhir"
+//        mockkObject(AuthenticatedClaims)
+//        val mockDb = mockk<DatabaseAccess>()
+//        every { mockDb.fetchReportFile(any()) } returns reportFile
+//        mockkClass(BlobAccess::class)
+//        mockkObject(BlobAccess.Companion)
+//        every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection"
+//        val blobConnectionInfo = mockk<BlobAccess.BlobContainerMetadata>()
+//        every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata"
+//        every { BlobAccess.downloadBlobAsByteArray(any<String>()) } returns report.toByteArray(Charsets.UTF_8)
+//        val downloadReport = DownloadReport()
+//        downloadReport.databaseAccess = mockDb
+//
+//        val result = downloadReport.test(
+//            "-r ${UUID.randomUUID()} -e local --remove-pii true",
+//            ansiLevel = AnsiLevel.TRUECOLOR
+//        )
+//
+//        assert(result.output.contains("MESSAGE OUTPUT"))
+//    }
+//
+//    @Test
+//    fun `valid access token, report found, asked for no removal on prod`() {
+//        val reportFile = ReportFile()
+//        reportFile.bodyUrl = "fakeurl.fhir"
+//        mockkObject(AuthenticatedClaims)
+//        val mockDb = mockk<DatabaseAccess>()
+//        every { mockDb.fetchReportFile(any()) } returns reportFile
+//        mockkClass(BlobAccess::class)
+//        mockkObject(BlobAccess.Companion)
+//        every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection"
+//        val blobConnectionInfo = mockk<BlobAccess.BlobContainerMetadata>()
+//        every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata"
+//        every { BlobAccess.downloadBlobAsByteArray(any<String>()) } returns report.toByteArray(Charsets.UTF_8)
+//        every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile
+//        mockkObject(CommandUtilities)
+//        every { CommandUtilities.isApiAvailable(any(), any()) } returns true
+//        val downloadReport = DownloadReport()
+//        downloadReport.databaseAccess = mockDb
+//
+//        val result = downloadReport.test(
+//            "-r ${UUID.randomUUID()} -e prod --remove-pii false",
+//            ansiLevel = AnsiLevel.TRUECOLOR
+//        )
+//
+//        assert(result.stderr.isNotBlank())
+//    }
+//
+//    @Test
+//    fun `valid access token, report found, no PII removal`() {
+//        val reportFile = ReportFile()
+//        reportFile.bodyUrl = "fakeurl.fhir"
+//        mockkObject(AuthenticatedClaims)
+//        val mockDb = mockk<DatabaseAccess>()
+//        every { mockDb.fetchReportFile(any()) } returns reportFile
+//        mockkClass(BlobAccess::class)
+//        mockkObject(BlobAccess.Companion)
+//        every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection"
+//        val blobConnectionInfo = mockk<BlobAccess.BlobContainerMetadata>()
+//        every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata"
+//        every { BlobAccess.downloadBlobAsByteArray(any<String>()) } returns report.toByteArray(Charsets.UTF_8)
+//        every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile
+//        val downloadReport = DownloadReport()
+//        downloadReport.databaseAccess = mockDb
+//
+//        val result = downloadReport.test(
+//            "-r ${UUID.randomUUID()} -e local --remove-pii false",
+//            ansiLevel = AnsiLevel.TRUECOLOR
+//        )
+//
+//        assert(result.output.contains("MESSAGE OUTPUT"))
+//    }
+//
+//    @Test
+//    fun `valid access token, report found, body URL not FHIR`() {
+//        val reportFile = ReportFile()
+//        reportFile.bodyUrl = "fakeurl.hl7"
+//        mockkObject(AuthenticatedClaims)
+//        val mockDb = mockk<DatabaseAccess>()
+//        every { mockDb.fetchReportFile(any()) } returns reportFile
+//        mockkConstructor(WorkflowEngine::class)
+//        every { anyConstructed<WorkflowEngine>().db } returns mockDb
+//        every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile
+//        val downloadReport = DownloadReport()
+//        downloadReport.databaseAccess = mockDb
+//
+//        val result = downloadReport.test(
+//            "-r ${UUID.randomUUID()} -e local --remove-pii true",
+//            ansiLevel = AnsiLevel.TRUECOLOR
+//        )
+//
+//        assert(result.stderr.contains("not fhir"))
+//    }
 }