build.gradle

buildscript {
  repositories {
    mavenLocal()
    maven {url 'http://developer.marklogic.com/maven2/'}
    jcenter()
  }

  dependencies {
    classpath "com.marklogic:ml-data-hub:5.5.1"
    classpath "com.marklogic:marklogic-unit-test-client:1.1.0"
  }
}

plugins {
  id 'java'
  id 'eclipse'
  id 'idea'

  // Gradle Properties plugin
  id 'net.saliman.properties' version '1.4.6'

  // Data Hub plugin
  id 'com.marklogic.ml-data-hub' version '5.5.1'

  //Data services needs these
  id 'com.marklogic.ml-development-tools' version '5.4.0'
}

repositories {
  maven {url "https://plugins.gradle.org/m2/"}
  mavenCentral()
  jcenter()
  maven {url 'http://developer.marklogic.com/maven2/'}
  if (project.hasProperty("testing")) {
    mavenLocal()
  }
}

configurations {
  mlcp
}

dependencies {
  mlcp "com.marklogic:mlcp:10.0.6.2"
  mlcp files("lib")
}

dependencies {
  // Ensures that the marklogic-unit-test modules are loaded into the modules database when the application is deployed
  mlBundle "com.marklogic:marklogic-unit-test-modules:1.0.0"

  testImplementation "com.marklogic:marklogic-data-hub-junit5:5.5.1"

  // Adds logging support for tests
  testImplementation "ch.qos.logback:logback-classic:1.2.3"

  mlcp "com.marklogic:mlcp:10.0.6.2"
  mlcp files("lib")

  implementation group: 'ca.uhn.hapi.fhir', name: 'hapi-fhir-base', version: '5.3.3'
  implementation group: 'ca.uhn.hapi.fhir', name: 'hapi-fhir-structures-r4', version: '5.3.3'
}

test {
  useJUnitPlatform()
}

task resetDBs(type: com.marklogic.gradle.task.ServerEvalTask) {
  javascript = 'xdmp.invoke("/envision/system/resetSystem.sjs")'
}

task loadReferenceDataStaging(type: com.marklogic.gradle.task.MlcpTask) {
  doFirst {
    database = mlStagingDbName
    port = mlStagingPort.toInteger()
  }
  classpath = configurations.mlcp
  command = "IMPORT"
  args = [
    "-input_file_path", "$project.projectDir/src/main/ml-data/referenceData/ValueSets",
    "-output_collections", "referenceData",
    "-output_permissions", "rest-reader,read,rest-writer,update",
    "-document_type", "json",
    "-output_uri_replace", ".*src/main/ml-data,''",
    "-modules_root", "/",
  ]
}

task loadMetaDataStaging(type: com.marklogic.gradle.task.MlcpTask) {
  doFirst {
    database = mlStagingDbName
    port = mlStagingPort.toInteger()
  }
  classpath = configurations.mlcp
  command = "IMPORT"
  args = [
    "-input_file_path", "$project.projectDir/src/main/ml-data/meta",
    "-output_collections", "metaData",
    "-output_permissions", "raw-data-reader,read,mes-writer,update",
    "-document_type", "json",
    "-output_uri_replace", ".*src/main/ml-data,''",
    "-modules_root", "/",
]
}

task loadMetaDataFinal(type: com.marklogic.gradle.task.MlcpTask) {
  doFirst {
    database = mlFinalDbName
    port = mlFinalPort.toInteger()
  }
  classpath = configurations.mlcp
  command = "IMPORT"
  args = [
    "-input_file_path", "$project.projectDir/src/main/ml-data/meta",
    "-output_collections", "metaData",
    "-output_permissions", "raw-data-reader,read,mes-writer,update",
    "-document_type", "json",
    "-output_uri_replace", ".*src/main/ml-data,''",
    "-modules_root", "/",
  ]
}

mlLoadData.dependsOn loadReferenceDataStaging
// In order to load additional reference and metadata, augment the existing mlLoadData task to look
// in these sub-directories and load them appropriately
mlLoadData.dependsOn loadMetaDataStaging
mlLoadData.dependsOn loadMetaDataFinal

ext.getOntologyLoadArgs = { ttlFile, outputGraph ->
  // Get custom assertion failure messages: https://stackoverflow.com/a/51447369
  assert 'You must provide a filename to load from.' && !ttlFile.equals('')
  assert "Invalid file extension for \"$ttlFile\". Expected \".ttl\" extension." && ttlFile.endsWith('.ttl')
  assert 'You must provide a graph name to output into.' && !outputGraph.equals('')

  return [
    '-input_file_path', ttlFile,
    '-output_graph', outputGraph,
    '-input_file_type', 'rdf'
  ]
}

task loadIcd10PcsOntology(type: com.marklogic.gradle.task.MlcpTask) {
  def ttlFile = "$project.projectDir/src/main/ml-data/ontologies/ICD10PCS.ttl"
  def ttlFileExists = file(ttlFile).exists()

  if (!ttlFileExists) {
    println "WARNING: Unable to find file \"$ttlFile\". Task `loadIcd10PcsOntology` will be a no-op."
  }

  onlyIf {
    ttlFileExists
  }

  database = mlFinalDbName
  port = mlFinalPort.toInteger()

  classpath = configurations.mlcp
  command = 'IMPORT'
  args = getOntologyLoadArgs(ttlFile, 'icd-10-pcs-ontology')
}

task loadIcd10CmOntology(type: com.marklogic.gradle.task.MlcpTask) {
  def ttlFile = "$project.projectDir/src/main/ml-data/ontologies/ICD10CM.ttl"
  def ttlFileExists = file(ttlFile).exists()

  if (!ttlFileExists) {
    println "WARNING: Unable to find file \"$ttlFile\". Task `loadIcd10CmOntology` will be a no-op."
  }

  onlyIf {
    ttlFileExists
  }

  database = mlFinalDbName
  port = mlFinalPort.toInteger()

  classpath = configurations.mlcp
  command = 'IMPORT'
  args = getOntologyLoadArgs(ttlFile, 'icd-10-cm-ontology')
}

task normalizeSnomedCTOntology(type: com.marklogic.gradle.task.ServerEvalTask) {
  xquery = 'xdmp:invoke("/lib/transformers/addSkosCoreNotationToSnomedOntology.sjs", (), map:entry("database", xdmp:database("' + mlFinalDbName + '")))'
}

task loadSnomedCTOntology(type: com.marklogic.gradle.task.MlcpTask) {
  def ttlFile = "$project.projectDir/src/main/ml-data/ontologies/SNOMED-CT.ttl"
  def ttlFileExists = file(ttlFile).exists()

  if (!ttlFileExists) {
    println "WARNING: Unable to find file \"$ttlFile\". Task `loadSnomedCTOntology` will be a no-op."
  }

  onlyIf {
    ttlFileExists
  }

  database = mlFinalDbName
  port = mlFinalPort.toInteger()

  classpath = configurations.mlcp
  command = 'IMPORT'
  args = getOntologyLoadArgs(ttlFile, 'snomed-ct-ontology')

  finalizedBy {
    normalizeSnomedCTOntology
  }
}

task loadOntologies {
  dependsOn loadIcd10PcsOntology
  dependsOn loadIcd10CmOntology
  dependsOn loadSnomedCTOntology
}

task ingestClaims (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest claims raw data"
  flowName = "Claim"
  steps = ["1"]
  inputFilePath="data/synthea/csv/claims/"
  inputFileType="csv"
}

task ingestClaimsSmaller (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest smaller claims raw data"
  flowName = "Claim"
  steps = ["1"]
  inputFilePath="data/synthea/csv/claims-smaller/"
  inputFileType="csv"
}

task ingestClaimLines(type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest claim transactions raw data"
  flowName = "ClaimTransaction"
  steps = ["1"]
  inputFilePath="data/synthea/csv/claims_transactions/"
  inputFileType="csv"
}

task ingestClaimLinesSmaller(type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest smaller claim transactions raw data"
  flowName = "ClaimTransaction"
  steps = ["1"]
  inputFilePath="data/synthea/csv/claims_transactions-smaller/"
  inputFileType="csv"
}

task ingestPatients (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest patient/member raw data"
  flowName = "Patient"
  steps = ["1"]
  inputFilePath="data/synthea/csv/patients/"
  inputFileType="csv"
}

task ingestProviders (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest provider raw data"
  flowName = "Provider"
  steps = ["1"]
  inputFilePath="data/synthea/csv/providers/"
  inputFileType="csv"
}

task ingestOrganizations (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest organization raw data"
  flowName = "Organization"
  steps = ["1"]
  inputFilePath="data/synthea/csv/organizations/"
  inputFileType="csv"
}

task ingestPayers (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest payer raw data"
  flowName = "Payor"
  steps = ["1"]
  inputFilePath="data/synthea/csv/payers/"
  inputFileType="csv"
}

task ingestSanctions (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Ingest practitioner sanctions"
  flowName = "Provider"
  steps = ["2"]
  inputFilePath="data/sanctions/"
  inputFileType="xml"
}

task ingest {
  dependsOn ingestClaims
  dependsOn ingestClaimLines
  dependsOn ingestPatients
  dependsOn ingestProviders
  dependsOn ingestOrganizations
  dependsOn ingestPayers
  dependsOn ingestSanctions
}

task ingestSmaller {
  dependsOn ingestClaimsSmaller
  dependsOn ingestClaimLinesSmaller
  dependsOn ingestPatients
  dependsOn ingestProviders
  dependsOn ingestOrganizations
  dependsOn ingestPayers
  dependsOn ingestSanctions
}

task harmonizePatients (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Harmonize patient data"
  flowName = "Patient"
  steps = ["2","3","4","5","6"]
}
task harmonizeOrganizations (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Harmonize organization data"
  flowName = "Organization"
  steps = ["2","3"]
}
task harmonizeProviders (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Harmonize provider data"
  flowName = "Provider"
  steps = ["3","4","5"]
}
task harmonizeClaims (type: com.marklogic.gradle.task.RunFlowTask) {
  description = "Harmonize claim data"
  flowName = "Claim"
  steps = ["2"]
}

task harmonizeAll {
  dependsOn harmonizePatients
  dependsOn harmonizeOrganizations
  dependsOn harmonizeProviders
  dependsOn harmonizeClaims
}

ext.getRedactedExportArgs = { directoryFilter, redactions, outputPath ->
  return [
    '-mode', 'local',
    '-output_file_path', outputPath, // Provided via gradle.properties or gradle-local.properties
    '-directory_filter', directoryFilter,
    '-redaction', redactions,
    '-max_split_size', '17500'
  ]
}

task exportRedactedPatients (type: com.marklogic.gradle.task.MlcpTask) {
  if (patientExportOutputDir.equals('')) {
    println "WARNING: No `patientExportOutputDir` configured in gradle properties. Task `exportRedactedPatients` will be a no-op."
  }
  onlyIf {
    !patientExportOutputDir.equals('')
  }
  doFirst {
    database = mlFinalDbName
    port = mlFinalPort.toInteger()
  }
  classpath = configurations.mlcp
  command = 'EXPORT'
  args = getRedactedExportArgs('/patient/', 'member-redaction', patientExportOutputDir)
}

task exportRedactedClaims (type: com.marklogic.gradle.task.MlcpTask) {
  if (claimExportOutputDir.equals('')) {
    println "WARNING: No `claimExportOutputDir` configured in gradle properties. Task `exportRedactedClaims` will be a no-op."
  }
  onlyIf {
    !claimExportOutputDir.equals('')
  }
  doFirst {
    database = mlFinalDbName
    port = mlFinalPort.toInteger()
  }
  classpath = configurations.mlcp
  command = 'EXPORT'
  args = getRedactedExportArgs('/claim/', 'claim-redaction', claimExportOutputDir)
}

task exportRedactedData {
  dependsOn exportRedactedPatients
  dependsOn exportRedactedClaims
}