build.gradle

buildscript {
    repositories {
        mavenLocal()
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
        maven { url "https://grobid.s3.eu-west-1.amazonaws.com/repo/" }
    }
    dependencies {
        classpath 'gradle.plugin.org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.12.0'
        classpath "gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0"
    }
}

plugins {
    id 'com.github.johnrengelman.shadow' version '7.0.0'
    id "de.undercouch.download" version "4.1.1"
}

apply plugin: 'jacoco'

jacoco {
    toolVersion = '0.8.8'
}

apply plugin: 'java-library'
apply plugin: 'base'
apply plugin: 'com.github.kt3k.coveralls'

group = "org.grobid.datastet"
version = '0.8.1'

description = """datastet"""

sourceCompatibility = 1.11
targetCompatibility = 1.11

import org.apache.tools.ant.taskdefs.condition.Os

tasks.withType(JavaCompile) {
    options.encoding = 'UTF-8'
}

repositories {
    mavenLocal()
    mavenCentral()
    maven { url "https://grobid.s3.eu-west-1.amazonaws.com/repo/" }
    maven {
        url new File(rootProject.rootDir, "lib")
    }
}

ext {
    // treating them separately, these jars will be flattened into grobid-core.jar on installing,
    // to avoid missing dependencies from the projects that include grobid-core (see 'jar' taskin grobid-core)
    //localLibs = ['org/dkpro/statistics/dkpro-statistics-agreement/2.2.0-SNAPSHOT/dkpro-statistics-agreement-2.2.0-SNAPSHOT.jar']
    localLibs = []
}

dependencies {

    // packaging local libs inside output jar
    //implementation fileTree(dir: new File(rootProject.rootDir, 'lib'), include: localLibs)

    implementation group: 'net.arnx', name: 'jsonic', version: '1.3.10'

    //Logging
    implementation 'org.slf4j:slf4j-api:1.7.30'

    //Apache commons
    implementation group: 'commons-pool', name: 'commons-pool', version: '1.6'
    implementation group: 'commons-io', name: 'commons-io', version: '2.5'
    //implementation group: 'commons-logging', name: 'commons-logging', version: '1.2'
    implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3'
    implementation group: 'org.apache.httpcomponents', name: 'httpmime', version: '4.5.3'
    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.6'
    implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.1'
    implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.5'

    implementation group: 'com.google.guava', name: 'guava', version: '28.2-jre'

    //Dropwizard
    implementation "io.dropwizard:dropwizard-core:1.3.23"
    implementation "io.dropwizard:dropwizard-assets:1.3.23"
    implementation "com.hubspot.dropwizard:dropwizard-guicier:1.3.5.0"
    implementation "io.dropwizard:dropwizard-testing:1.3.23"
    implementation "io.dropwizard:dropwizard-forms:1.3.23"
    implementation "io.dropwizard:dropwizard-client:1.3.23"
    implementation "io.dropwizard:dropwizard-auth:1.3.23"
    implementation "io.dropwizard.metrics:metrics-core:4.0.0"
    implementation "io.dropwizard.metrics:metrics-servlets:4.0.0"

    //Parsing xml/json
    //Parsing xml/json
    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.1'
    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1'
    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.10.1'
    implementation group: 'xom', name: 'xom', version: '1.3.2'
    implementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.0'
    implementation group: 'com.sun.xml.bind', name: 'jaxb-core', version: '2.3.0'
    implementation group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.0'

    implementation group: 'javax.activation', name: 'activation', version: '1.1.1'
    implementation group: 'asm', name: 'asm', version: '3.3.1'

    implementation 'black.ninia:jep:4.0.2'
    implementation "joda-time:joda-time:2.9.9"
    implementation "org.apache.lucene:lucene-analyzers-common:4.5.1"
    implementation group: 'org.jruby', name: 'jruby-complete', version: '9.2.13.0'
    implementation "org.apache.pdfbox:pdfbox:2.0.18"

    //implementation 'org.jvnet.mimepull:mimepull:1.6'

    //providedImplementation group: 'javax.servlet', name: 'servlet-api', version: '2.4'
    //providedImplementation group: 'javax.servlet.jsp', name: 'jsp-api', version: '2.0'
    implementation group: 'org.jvnet', name: 'mimepull', version: '1.6'
    //implementation group: 'com.sun.jersey.contribs', name: 'jersey-multipart', version: '1.8'
    //implementation group: 'javax.ws.rs', name: 'jsr311-api', version: '1.1.1'

    // for sentence segmentation only
    implementation 'org.apache.opennlp:opennlp-tools:1.9.1'

    //Grobid
    implementation group: 'org.grobid', name: 'grobid-core', version: '0.8.1'
    implementation group: 'org.grobid', name: 'grobid-trainer', version: '0.8.1'

    //Tests
    testImplementation group: 'junit', name: 'junit', version: '4.12'
    testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3'
}

configurations.all { 
    resolutionStrategy {
        force 'xml-apis:xml-apis:1.4.01'
    }
    
    exclude group: 'org.slf4j', module: "slf4j-log4j12"
    exclude group: 'org.slf4j', module: "slf4j-jdk14"
    exclude group: 'log4j', module: "log4j"

    //if (project.gradle.startParameter.taskNames.contains('run'))
    exclude group: 'org.slf4j', module: "log4j-over-slf4j"
}

def libraries = ""
if (Os.isFamily(Os.FAMILY_MAC)) {
    if (Os.OS_ARCH.equals("aarch64")) {
        libraries = "${file("../grobid-home/lib/mac_arm-64").absolutePath}"    
    } else {
        libraries = "${file("../grobid-home/lib/mac-64").absolutePath}"
    }
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
    libraries = "${file("../grobid-home/lib/lin-64/jep").absolutePath}:" +
        "${file("../grobid-home/lib/lin-64").absolutePath}:"
} else  {
    throw new RuntimeException("Unsupported platform!")
}

configurations.implementation.setCanBeResolved(true)

test {
    testLogging.showStandardStreams = true
    exclude '**/**IntegrationTest**'

    if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
        jvmArgs "--add-opens", "java.base/java.util.stream=ALL-UNNAMED", 
        "--add-opens", "java.base/java.io=ALL-UNNAMED"
    }
    systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}

// return the default value if the property has not been specified in command line
ext.getArg = { propName, defaultVal ->
    return project.hasProperty(propName) ? project.getProperty(propName) : defaultVal;
}

import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

apply plugin: 'java'
apply plugin: 'maven-publish'

publishing {
    publications {
        mavenJava(MavenPublication) {
            from components.java
        }
    }
    repositories {
        mavenLocal()
    }
}

task install(dependsOn: publishToMavenLocal)

task mainJar(type: ShadowJar) {
    zip64 true
    from sourceSets.main.output

    configurations = [project.configurations.runtimeClasspath]

    from {
        project.configurations.implementation.collect {
            it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
        }
    }
}

shadowJar {
    classifier = 'onejar'
    mergeServiceFiles()
    zip64 true
    manifest {
        attributes 'Main-Class': 'org.grobid.core.main.batch.DatastetMain'
    }

    configurations = [project.configurations.runtimeClasspath]

    from {
        project.configurations.implementation.collect {
            it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
        }
    }
}

jar {
    dependsOn mainJar
    enabled true
}

tasks.withType(Tar){
    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}

tasks.withType(Zip){
    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}

artifacts {
    archives jar
    archives shadowJar
}

task installModels(type: Copy) {
    from "${rootDir}/resources/models/"
    include "**"
    into "${rootDir}/../grobid-home/models/"

    doLast {
        download {
            src "https://grobid.s3.amazonaws.com/dataseer-binary_bert-0.3.1.zip"
            dest "${rootDir}/../grobid-home/models/"
            overwrite false
        }
        ant.unzip(src: "${rootDir}/../grobid-home/models/dataseer-binary_bert-0.3.1.zip", dest: "${rootDir}/../grobid-home/models/")
        
        download {
            src "https://grobid.s3.amazonaws.com/dataseer-first_bert-0.3.1.zip"
            dest "${rootDir}/../grobid-home/models/"
            overwrite false
        }
        ant.unzip(src: "${rootDir}/../grobid-home/models/dataseer-first_bert-0.3.1.zip", dest: "${rootDir}/../grobid-home/models/")
        
        download {
            src "https://grobid.s3.amazonaws.com/dataseer-reuse_bert-0.3.1.zip"
            dest "${rootDir}/../grobid-home/models/"
            overwrite false
        }
        ant.unzip(src: "${rootDir}/../grobid-home/models/dataseer-reuse_bert-0.3.1.zip", dest: "${rootDir}/../grobid-home/models/")

        download {
            src "https://grobid.s3.amazonaws.com/datasets-BERT_CRF-0.3.2.zip"
            dest "${rootDir}/../grobid-home/models/"
            overwrite false
        }
        ant.unzip(src: "${rootDir}/../grobid-home/models/datasets-BERT_CRF-0.3.2.zip", dest: "${rootDir}/../grobid-home/models/")

        download {
            src "https://grobid.s3.amazonaws.com/context_bert-0.3.2.zip"
            dest "${rootDir}/../grobid-home/models/"
            overwrite false
        }
        ant.unzip(src: "${rootDir}/../grobid-home/models/context_bert-0.3.2.zip", dest: "${rootDir}/../grobid-home/models/")
        
        download {
            src "https://grobid.s3.amazonaws.com/context_bert_used-0.3.2.zip"
            dest "${rootDir}/../grobid-home/models/"
            overwrite false
        }
        ant.unzip(src: "${rootDir}/../grobid-home/models/context_bert_used-0.3.2.zip", dest: "${rootDir}/../grobid-home/models/")
        
        download {
            src "https://grobid.s3.amazonaws.com/context_bert_creation-0.3.2.zip"
            dest "${rootDir}/../grobid-home/models/"
            overwrite false
        }
        ant.unzip(src: "${rootDir}/../grobid-home/models/context_bert_creation-0.3.2.zip", dest: "${rootDir}/../grobid-home/models/")
        
        download {
            src "https://grobid.s3.amazonaws.com/context_bert_shared-0.3.2.zip"
            dest "${rootDir}/../grobid-home/models/"
            overwrite false
        }
        ant.unzip(src: "${rootDir}/../grobid-home/models/context_bert_shared-0.3.2.zip", dest: "${rootDir}/../grobid-home/models/")
    }
}

//tasks.withType(JavaCompile) {
//    options.compilerArgs << "-Xlint:deprecation"
//    options.compilerArgs << "-Xlint:unchecked"
//}

wrapper {
    gradleVersion "7.2"
}

// Custom Tasks

// Run like this: ./gradlew train_dataseer -PgH=/path/grobid/home
task(train_dataseer, dependsOn: 'classes', type: JavaExec, group: 'training') {
    mainClass = 'org.grobid.trainer.DataseerTrainerRunner'
    classpath = sourceSets.main.runtimeClasspath
    args '0', 'dataseer', getArg('gH', '../grobid-home'), getArg('t', '0')
    if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
    } else {
        jvmArgs '-Xmx3072m'
    }
    systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
//    jvmArgs '-Xms2g', '-Xmx8g'
}

// Run like this: ./gradlew train_dataseer -PgH=/path/grobid/home
//task(train_dataseer, dependsOn: 'classes', type: JavaExec, group: 'training') {
//    mainClass = 'org.grobid.trainer.DataseerTrainer'
//    classpath = sourceSets.main.runtimeClasspath
//    args '0', 'dataseer', getArg('gH', '../grobid-home'), getArg('b', ''),
//    jvmArgs '-Xms2g', '-Xmx8g'
//}

// Run like this: ./gradlew eval_dataseer -PgH=/path/grobid/home
task(eval_dataseer, dependsOn: 'classes', type: JavaExec, group: 'evaluation') {
    mainClass = 'org.grobid.trainer.DataseerTrainerRunner'
    classpath = sourceSets.main.runtimeClasspath
    args '1', 'dataseer', getArg('gH', '../grobid-home')
    if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
    } else {
        jvmArgs '-Xmx3072m'
    }
    systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
//    jvmArgs '-Xms2g', '-Xmx8g'
}

// Run like this: ./gradlew eval_dataseer_split -PgH=/path/grobid/home -Ps=0.8 -Pt=10
task(eval_dataseer_split, dependsOn: 'classes', type: JavaExec, group: 'evaluation') {
    mainClass = 'org.grobid.trainer.DataseerTrainerRunner'
    classpath = sourceSets.main.runtimeClasspath
    args '2', 'dataseer', getArg('gH', '../grobid-home'), getArg('s', 0.9), getArg('t', '0')
    //jvmArgs '-Xms2g', '-Xmx8g'
    if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
        jvmArgs '-Xms2g', '-Xmx8g', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
    } else {
        jvmArgs '-Xms2g', '-Xmx8g'
    }
    systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}

// Run like this: ./gradlew eval_dataseer_nfold -PgH=/path/grobid/home -Pt=10
task(eval_dataseer_nfold, dependsOn: 'classes', type: JavaExec, group: 'evaluation') {
    mainClass = 'org.grobid.trainer.DataseerTrainerRunner'
    classpath = sourceSets.main.runtimeClasspath
    args '3', 'dataseer', getArg('gH', '../grobid-home'), getArg('n', 10), getArg('t', '0')
    if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
        jvmArgs '-Xms2g', '-Xmx8g', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
    } else {
        jvmArgs '-Xms2g', '-Xmx8g'
    }
    systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}

// Run like this: ./gradlew annotated_corpus_generator_csv -Pfull=/path/input/fulltext -Ppdf=/path/input/pdf -Pcsv=/path/csv -Pxml=/output/directory
task(annotated_corpus_generator_csv, dependsOn: 'classes', type: JavaExec, group: 'import') {
    mainClass = 'org.grobid.trainer.AnnotatedCorpusGeneratorCSV'
    classpath = sourceSets.main.runtimeClasspath
    args getArg('full', ''), getArg('pdf', ''), getArg('csv', ''), getArg('xml', 'resources/dataset/dataseer/corpus/')
    if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
        jvmArgs '-Xms2g', '-Xmx8g', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
    } else {
        jvmArgs '-Xms2g', '-Xmx8g'
    }
    systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
}

//////////
// service
apply plugin: 'application'
application {
    mainClassName = 'org.grobid.service.DatastetApplication'

    run {
        if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
            jvmArgs "--add-opens", "java.base/java.lang=ALL-UNNAMED"
        }
        workingDir = rootProject.rootDir
        def javaLibraryPath = "${System.getProperty('java.library.path')}:" + libraries
        systemProperty "java.library.path", javaLibraryPath

        args = ['server', 'resources/config/config.yml']
    }
}