build.gradle

/*
 *  Copyright OpenSearch Contributors
 *  SPDX-License-Identifier: Apache-2.0
 */

import org.opensearch.gradle.test.RestIntegTestTask
import org.apache.tools.ant.taskdefs.condition.Os

buildscript {
    ext {
        // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set
        // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT
        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
        version_qualifier = System.getProperty("build.version_qualifier", "")
        opensearch_group = "org.opensearch"
    }

    // This isn't applying from repositories.gradle so repeating git diff it here
    repositories {
        mavenLocal()
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
    }

    dependencies {
        classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}"
    }
}

//****************************************************************************/
// Build configurations
//****************************************************************************/

plugins {
    id 'java-library'
    id 'java-test-fixtures'
    id 'idea'
    id 'jacoco'
    id "com.diffplug.spotless" version "6.3.0" apply false
    id 'io.freefair.lombok' version '6.4.3'
}

apply from: 'gradle/formatting.gradle'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.rest-test'
apply plugin: 'opensearch.pluginzip'
apply plugin: 'opensearch.repositories'

ext {
    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
    projectSubstitutions = [:]
}

allprojects {

    group = 'org.opensearch'
    version = opensearch_version.tokenize('-')[0] + '.0'
    if (version_qualifier) {
        version += "-${version_qualifier}"
    }
    if (isSnapshot) {
        version += "-SNAPSHOT"
    }
    apply from: rootProject.file('build-tools/repositories.gradle').absoluteFile
    plugins.withId('java') {
        sourceCompatibility = targetCompatibility = "11"
    }

    afterEvaluate {
        project.dependencyLicenses.enabled = false
        project.thirdPartyAudit.enabled = false
        project.loggerUsageCheck.enabled = false
        project.forbiddenApis.ignoreFailures = true
        project.forbiddenPatterns {
            setEnabled(false)
        }
        project.testingConventions.enabled = false
        project.validateNebulaPom.enabled = false
        project.licenseFile = rootProject.file('LICENSE.txt')
        project.noticeFile = rootProject.file('NOTICE.txt')
        project.forbiddenApis.ignoreFailures = true
    }
}

publishing {
    publications {
        pluginZip(MavenPublication) { publication ->
            pom {
                name = "opensearch-knn"
                description = "OpenSearch k-NN plugin"
                groupId = "org.opensearch.plugin"
                licenses {
                    license {
                        name = "The Apache License, Version 2.0"
                        url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
                    }
                }
                developers {
                    developer {
                        name = "OpenSearch"
                        url = "https://github.com/opensearch-project/k-NN"
                    }
                }
            }
        }
    }
}

compileJava {
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}
compileTestJava {
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}

def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
def usingMultiNode = project.properties.containsKey('numNodes')
// Only apply jacoco test coverage if we are running a local single node cluster
def integTestDependOnJniLib = false
if (!usingRemoteCluster) {
    integTestDependOnJniLib = true
    if (!usingMultiNode) {
        apply from: 'build-tools/knnplugin-coverage.gradle'
    }
}

jacoco {
    toolVersion = "0.8.7"
}

check.dependsOn spotlessCheck
check.dependsOn jacocoTestReport

opensearchplugin {
    name 'opensearch-knn'
    // zip file name and plugin name in ${opensearch.plugin.name} read by OpenSearch when plugin loading
    description 'OpenSearch k-NN plugin'
    classname 'org.opensearch.knn.plugin.KNNPlugin'
    extendedPlugins = ['lang-painless']
    licenseFile = rootProject.file('LICENSE.txt')
    noticeFile = rootProject.file('NOTICE.txt')
}

tasks.named("integTest").configure {
    it.dependsOn(project.tasks.named("bundlePlugin"))
}

task release(type: Copy, group: 'build') {
    dependsOn project.tasks.build
    from(zipTree(project.tasks.bundlePlugin.outputs.files.getSingleFile()))
    into "build/plugins/opensearch-knn"
    includeEmptyDirs = false
    eachFile { it.path = it.path - "opensearch/" }
}


//****************************************************************************/
// Dependencies
//****************************************************************************/
dependencies {
    api "org.opensearch:opensearch:${opensearch_version}"
    compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}"
    api group: 'com.google.guava', name: 'failureaccess', version:'1.0.1'
    api group: 'com.google.guava', name: 'guava', version:'30.0-jre'
    api group: 'commons-lang', name: 'commons-lang', version: '2.6'
    testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}"
}


def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()

task windowsPatches(type:Exec) {
    commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1"
}

task cmakeJniLib(type:Exec) {
    workingDir 'jni'
    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
        dependsOn windowsPatches
        commandLine 'cmake', '.', "-G", "Unix Makefiles", "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll"
    }
    else {
        commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}"
    }
}

task buildJniLib(type:Exec) {
    dependsOn cmakeJniLib
    workingDir 'jni'
    commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss'
}

test {
    dependsOn buildJniLib
    systemProperty 'tests.security.manager', 'false'
    systemProperty "java.library.path", "$rootDir/jni/release"
    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
        // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables
        environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
    }
}

def _numNodes = findProperty('numNodes') as Integer ?: 1
integTest {
    if (integTestDependOnJniLib) {
        dependsOn buildJniLib
    }
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
    systemProperty "java.library.path", "$rootDir/jni/release"
    // allows integration test classes to access test resource from project root path
    systemProperty('project.root', project.rootDir.absolutePath)

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    doFirst {
        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
        // use longer timeouts for requests.
        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
        systemProperty 'cluster.debug', isDebuggingCluster
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The -Ddebug.es option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000'
    }
}

testClusters.integTest {
    testDistribution = "ARCHIVE"
    plugin(project.tasks.bundlePlugin.archiveFile)
    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
        // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables
        environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
    }

    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug opensearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("cluster.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=${debugPort}")
            debugPort += 1
        }
    }
    systemProperty("java.library.path", "$rootDir/jni/release")
}

task integTestRemote(type: RestIntegTestTask) {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    systemProperty 'cluster.number_of_nodes', "${_numNodes}"

    systemProperty 'tests.security.manager', 'false'

    // Run tests with remote cluster only if rest case is defined
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.knn.*IT"
        }
    }
}

run {
    useCluster project.testClusters.integTest
    dependsOn buildJniLib
    doFirst {
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }
}

// updateVersion: Task to auto increment to the next development iteration
task updateVersion {
    onlyIf { System.getProperty('newVersion') }
    doLast {
        ext.newVersion = System.getProperty('newVersion')
        println "Setting version to ${newVersion}."
        // String tokenization to support -SNAPSHOT
        // Include the required files that needs to be updated with new Version
        ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) {
            fileset(dir: projectDir) {
                // Include the required files that needs to be updated with new Version
                include(name: ".github/workflows/backwards_compatibility_tests_workflow.yml")
            }
        }
        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
    }
}