build.gradle

import java.text.Collator

apply plugin: 'java'
apply plugin: 'groovy'
apply plugin: 'eclipse'

compileJava.options.encoding = 'UTF-8'

version=new File("${projectDir}/VERSION").text.trim()
group = 'org.dict_uk'

String langCode="uk"

String affixDir="data/affix"
String dataDir="data/dict"

String outputDir="out"
String prevDir="../../out/prev"


repositories {
    mavenCentral()
}

//configurations {
//    groovyScript
//}

dependencies {
    implementation group: 'org.codehaus.groovy', name: 'groovy-all', version: '3.0.+'
    implementation group: 'org.codehaus.groovy', name: 'groovy-cli-picocli', version: '3.0.+'
    implementation 'info.picocli:picocli:4.3.2'

    implementation group: 'commons-cli', name: 'commons-cli', version: '1.4'
    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'

    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.+'
    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.+'

//    groovyScript localGroovy()
}

//compileGroovy {
//    exclude '**/tools/other/*.groovy'
//}

eclipse {
  classpath {
    defaultOutputDir = file('build')
  }
}


//
// Auxilliary targets
//

task reverseVerbs (type: JavaExec, dependsOn: classes) {
    def inputFile = "$affixDir/v.aff"
    inputs.file inputFile
    outputs.file "$affixDir/vr.aff"

    classpath = sourceSets.main.runtimeClasspath
    main = "org.dict_uk.expand.VerbReverse"
    args file(inputFile), file(outputs.files[0])

//    standardInput = file(inputFile).newInputStream()
//    standardOutput = file(outputs.files[0]).newOutputStream()
}

task reverseAdvp (type: JavaExec, dependsOn: classes) {
    def inputFile = "$affixDir/v_advp.aff"
    inputs.file inputFile
    outputs.file "$affixDir/vr_advp.aff"

    classpath = sourceSets.main.runtimeClasspath
    main = "org.dict_uk.expand.VerbReverse"

    args file(inputFile), file(outputs.files[0])
}



task autogen (type: JavaExec, dependsOn: classes) {
	classpath = sourceSets.main.runtimeClasspath
	main = "org.dict_uk.tools.Autogen"

	String inputFile1 = "$dataDir/base.lst"
    String inputFile2 = "$dataDir/base-compound.lst"
	String inputFile3 = "$dataDir/base-compound_1992.lst"
	
    inputs.file inputFile1
    inputs.file inputFile2
    inputs.file inputFile3
    outputs.file "$dataDir/base-autogen.lst"

	args inputFile1, inputFile2, inputFile3, "$dataDir/base-autogen.lst"
}

task reverseAffix (dependsOn: [reverseVerbs, reverseAdvp]) {
}


task sortDict(type: JavaExec) {
	classpath = sourceSets.main.runtimeClasspath
	main = "org.dict_uk.tools.Sort"

	workingDir = file("data/dict")
	
	def files = "alt.lst base-abbr.lst base.lst base-compound.lst colors.lst dot-abbr.lst"
	files += " geo-ukr-hydro.lst geo-other.lst"  //  geo-ukr-koatuu.lst
	files += " slang.lst twisters.lst invalid.lst invalid-compound.lst names-anim.lst names-other.lst arch.lst"
	files += " names-composite.lst base-composite.lst subst.lst vulgar.lst"

	args files.split()
}


task dups {
  doLast {
	def dictDir = file("data/dict")
	
	def files = dictDir.listFiles().findAll { it.name.endsWith('.lst') \
	    && ! (it.name in ['dot-abbr.lst', 'geo-ukr-hydro.lst', 'geo-ukr-koatuu.lst']) }
	
	def lines = files.collect {  it.readLines()  }.flatten() \
	    .collect { it
	        .trim()
	        .replaceAll(/\s*#.*/, '')
	        .replaceAll(/( \/[a-z0-9]+)(\s+|$)/, '$1... ')
	        .replaceAll(/\.[^<.: ]+/, '...')
	        .replaceAll(/:&adjp:(actv|pasv)(:(imperf|perf))+/, '') } \
	    .grep { it && ! it.startsWith("+cs") }

    lines = lines.grep { ! it.startsWith('#') }

    def dups = lines.countBy{it}.grep{it.value > 1 }.collect{it.key}

    java.text.Collator coll = java.text.Collator.getInstance(new Locale("uk", "UA"));
    coll.setStrength(java.text.Collator.IDENTICAL)
    coll.setDecomposition(java.text.Collator.NO_DECOMPOSITION)


    println dups.toSorted(coll).join("\n")
    println "Found ${dups.size} dups"
  }
}

task checkReplacements {
  doLast {
	def dictDir = file("data/dict")
	
	def files = dictDir.listFiles().findAll { it.name.endsWith('.lst') || it.name.endsWith('replace.txt') }
	
	def replWords = files.collect { it.text.split("\n")  } \
	    .flatten() \
	    .findAll { it.contains(" #> ") }
	    .collect { 
			def repls = it.split(" #> ")[1].trim().split(/[;, \|]+/)
			def word = it.split(' ', 2)[0]
			if( word in repls ) {
				println "Replacement is same as word: $word"
			} 
			repls
		}
	    .flatten()
	    .collect { it.replaceAll(/[()]/, '').replaceFirst(/-таки$/, '') }
	    .findAll { it =~ /[а-яіїєґ]/ }
	    .unique().sort()

    def spellWords = new File("out/words_spell.txt").text.split('\n')
    spellWords += new File("data/dict/slang.lst").text.split('\n').collect{ it.replaceFirst(/ .*/, '') }
	spellWords += new File("data/dict/arch.lst").text.split('\n').collect{ it.replaceFirst(/ .*/, '') }
	
    println "Unique replacement words: ${replWords.size}"
    
    replWords.removeAll(spellWords)


    println "Unknown:\n" + replWords.join("\n")
    println "Total uknown: ${replWords.size}"
  }
}


task showExpandCommand {
  doLast {
    def opts = " --indent --mfl --stats --wordlist --time"
    opts += " -aff data/affix"
    opts += " -dict data/dict"

    def cmd = "groovy -cp "
    cmd += sourceSets.main.output.classesDir
    cmd += ":" + sourceSets.main.output.resourcesDir
    cmd += ":" + configurations.runtime.asFileTree.files.join(':')
//    cmd += " org.dict_uk.expand.ExpandAll"
    cmd += " src/main/groovy/org/dict_uk/expand/ExpandAll.groovy"
    cmd += opts
    
    println cmd
  }
}

//
// Testing targets
//

def getAdjustedText(filename) {
       new File(filename).text.replace("\r\n", "\n")
}


task testExpand(type: JavaExec, dependsOn: [reverseAffix, autogen, classes]) {
    classpath = sourceSets.main.runtimeClasspath
    main = "org.dict_uk.expand.ExpandAll"

    workingDir = file('test')

    def opts = " --indent --mfl --stats --wordlist"
    args "--aff", "../data/affix"
    args "--dict", "dict"
    args opts.split()

//    jvmArgs "-XX:+FlightRecorder -XX:StartFlightRecording=disk=true,dumponexit=true,filename=recording.jfr".split()

    doLast {
        def file="dict_corp_lt.txt"
        def noTestOutputChange = getAdjustedText("$workingDir/$file") == getAdjustedText("$workingDir/prev/$file")
        assert noTestOutputChange

        file="dict_corp_vis.txt"
        def noTestVisOutputChange = getAdjustedText("$workingDir/$file") == getAdjustedText("$workingDir/prev/$file")
        assert noTestVisOutputChange

//        def noTestTagsOutputChange = getAdjustedText("$workingDir/tags.txt") == getAdjustedText("$workingDir/prev/tags.txt")
//        assert noTestTagsOutputChange
    }
}


//
// Main targets
//

task expandForCorp(type: JavaExec, dependsOn: [reverseAffix, autogen/*, testExpand*/]) {
    classpath = sourceSets.main.runtimeClasspath
    main = "org.dict_uk.expand.ExpandAll"

    workingDir = file(outputDir)

    def opts = "--indent --mfl --stats --wordlist --time"
    // opts += "--removeWithRegex verb.*:inf:short"            // remove colloquial short verb forms

    args "--aff", "../data/affix"
    args "--dict", "../data/dict"
    args opts.split()

    jvmArgs "-Xms4000m -Xmx5000m".split()
    // JDK8
//    jvmArgs "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=dumponexit.jfr".split()
    // JDK11
//    jvmArgs "-XX:+FlightRecorder -XX:StartFlightRecording=disk=true,dumponexit=true,filename=recording.jfr".split()
//    jvmArgs "-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=n"
}

def get_diff_cmd(outputDir, file) {
    return "diff -u $outputDir/prev/$file $outputDir/$file > $outputDir/${file}.diff || echo 'File $file differs!'"
}


task diff(type: Exec) {
    def dictFile="dict_corp_lt.txt"

//    def cmd = get_diff_cmd(outputDir, dictFile)
//    cmd += "; " + get_diff_cmd(outputDir, 'words.txt')
//    cmd += "; " + get_diff_cmd(outputDir, 'words_spell.txt')
//    cmd += "; " + get_diff_cmd(outputDir, 'lemmas.txt')
//    cmd += "; " + get_diff_cmd(outputDir, 'tags.txt')

    dictFile="dict_corp_vis.txt"
    def cmd = get_diff_cmd(outputDir, dictFile)

    commandLine "sh", "-c", "${cmd}"
}

task expand(dependsOn: expandForCorp) {
}


task expandInteractive(type: JavaExec, dependsOn: reverseAffix) {
    classpath = sourceSets.main.runtimeClasspath
    main = "org.dict_uk.expand.Expand"

    workingDir = file(outputDir)
    standardInput = System.in

    def opts = "-f"
    if( ! project.hasProperty('flat') ) {
        opts += " --indent"
    }
    args "--aff", "../data/affix"
    args "--dict", "-"
    args opts.split()
}



task runTool(type: JavaExec, dependsOn: compileGroovy) {
    classpath = sourceSets.main.output + configurations.runtimeClasspath
    main = "groovy.ui.GroovyMain"
    args "src/tools/groovy/" + (project.hasProperty('tool') ? project.tool : '')
    if( project.hasProperty('file') ) {
        println "Reading word list from: " + project.file
        args project.file
    }
}


//
// deploy LT dict
//

task deployLtDict(type: GradleBuild) {
    dir = 'distr/morfologik-ukrainian'
    tasks = ['deployLtDict']
}


// windows have non-unicode encoding set by default
String osName = System.getProperty("os.name").toLowerCase();
if ( osName.contains("windows")) {
    if( ! "UTF-8".equals(System.getProperty("file.encoding")) ) {
        println "On Windows to get unicode handled correctly you need to set environment variable before running expand:"
        println "\tbash:"
        println "\t\texport JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF-8"
        println "\tcmd:"
        println "\t\t(change Font to 'Lucida Console' in cmd window properties)"
        println "\t\tchcp 65001"
        println "\t\tset JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF-8"
        println "\n\tNOTE: bash shell (e.g. git bash) is recommended: only in bash the output will handle all cyrillics and expandInteractive only supported in bash"

        // poor man's safety net (may work but still will not show all cyrillic characters)
        tasks.each { task ->
            if( task instanceof JavaExec ) {
                task.jvmArgs '-Dfile.encoding=UTF-8'
            }
        }
    }
}

task fatJar(type: Jar) {
    manifest {
        attributes 'Implementation-Title': 'dict_uk classes',
            'Implementation-Version': archiveVersion
    }
    archiveBaseName = project.name + '-all'
    from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
    with jar
}