Merge branch 'web-ui'

README.md

@@ -4,14 +4,28 @@
 
 Participation in [Solidabis code challenge 2019](https://koodihaaste.solidabis.com)
 
+Get a list of Caesar-ciphered strings, and determine whether they are Finnish language.
+The cipher employs characters a-z, å, ä, ö; extra characters are left as-is.
+Some strings may contain only a bullshit message, thus the program must determine if the
+strings are truly Finnish or not.
+
+The implementation compares each string and its deciphered variations against expected
+character distribution typical to Finnish language. The probability on being Finnish
+is calculated from Pearson's Chi-squared test result.
+
+The program is deployed to [Heroku]. In the web UI, candidate strings are displayed divided
+to *bullshit* and *no bullshit* sentences. *No bullshit* sentences are shown in Finnish,
+*bullshit* sentences are displayed in their original form.
+
+
 ## Testing commits and deploying
 
 - Always run tests before pushing with `./gradle test` and/or test build process
  with `./gradle clean build`
     - **JDK 12 is required**. See configuration from e.g. [this Stack Overflow answer](https://stackoverflow.com/a/21212790)
-    - For Heroku, this is already specified in [system.properties]
-- Test Heroku with `heroku local`
-- Increment version in [build.gradle.kts]
+    - For Heroku, this is already specified in [system.properties](system.properties)
+- Test Heroku with `./gradle stage` and `heroku local`
+- Increment version in [build.gradle.kts](build.gradle.kts)
 - Git commit
 - Push to GitHub master branch; Heroku automatic deployment takes care of the rest
     - Alternatively, push directly to heroku: `git push heroku master`

build.gradle.kts

@@ -14,7 +14,7 @@ plugins {
 }
 
 group = "fi.mkouhia.solidabis"
-version = "0.1.1"
+version = "0.2.0"
 
 application {
     mainClassName = "io.ktor.server.netty.EngineMain"

src/main/kotlin/fi/mkouhia/solidabis/bullshit/Application.kt

@@ -13,6 +13,11 @@ import io.ktor.thymeleaf.Thymeleaf
 import io.ktor.thymeleaf.ThymeleafContent
 import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver
 
+object Settings {
+    val secretUrl = "https://koodihaaste-api.solidabis.com/secret"
+
+}
+
 fun main(args: Array<String>): Unit = io.ktor.server.netty.EngineMain.main(args)
 
 fun Application.module() {
@@ -21,17 +26,11 @@ fun Application.module() {
             serializer = KotlinxSerializer()
         }
     }
-    runBlocking {
-        // Sample for making a HTTP Client request
-        /*
-        val message = client.post<JsonSampleClass> {
-            url("http://127.0.0.1:8080/path/to/endpoint")
-            contentType(ContentType.Application.Json)
-            body = JsonSampleClass(hello = "world")
-        }
-        */
+    val bullshitConnection = runBlocking {
+        BullshitConnection.fromSecretUrl(Settings.secretUrl, client)
     }
 
+
     install(Thymeleaf) {
         setTemplateResolver(ClassLoaderTemplateResolver().apply {
             prefix = "templates/thymeleaf/"
@@ -42,12 +41,9 @@ fun Application.module() {
 
     routing {
         get("/") {
-            call.respondText("HELLO WORLD!", contentType = ContentType.Text.Plain)
+            call.respond(ThymeleafContent("index", mapOf("bullshitConnection" to bullshitConnection)))
         }
 
-        get("/html-thymeleaf") {
-            call.respond(ThymeleafContent("index", mapOf("user" to ThymeleafUser(1, "user1"))))
-        }
     }
 }
 

src/main/kotlin/fi/mkouhia/solidabis/bullshit/Bullshit.kt

@@ -28,21 +28,21 @@ class Bullshit(val message: String) {
         return "Bullshit(message='$message')"
     }
 
-    /**
-     * TODO get best performing candidate
-     *
-     * @return best candidate, by lowest monogram P value
-     */
-    fun bestCandidate(): Candidate {
-        return candidates
+    /** Best candidate, by highest probability of being Finnish */
+    val bestCandidate: Candidate by lazy {
+        candidates
             .map {
-                it to it.monogramPValue
+                it to it.finnishProbability
             }
-            .toList()
             .sortedBy { (_, value) -> -value }[0]
             .first
     }
 
+    /** Best candidate is likely to be Finnish */
+    val isLikelyFinnish: Boolean by lazy { bestCandidate.isLikelyFinnish }
+    /** Probability of best candidate of being Finnish */
+    val finnishProbability: Double by lazy { bestCandidate.finnishProbability }
+
 
     /**
      * Rotate content by N characters
@@ -72,7 +72,7 @@ class Bullshit(val message: String) {
      * @param nChars number of characters forward
      * @return resulting character
      */
-    fun rotateChar(c: Char, nChars: Int): Char {
+    private fun rotateChar(c: Char, nChars: Int): Char {
         val charInd = when (c.toLowerCase()) {
             in 'a'..'z' -> c.toLowerCase() - 'a'
             'å' -> 26
@@ -94,6 +94,21 @@ class Bullshit(val message: String) {
         }
         return if (c.isUpperCase()) newChar.toUpperCase() else newChar
     }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as Bullshit
+
+        if (message != other.message) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        return message.hashCode()
+    }
 }
 
 /**
@@ -102,7 +117,7 @@ class Bullshit(val message: String) {
 class Candidate(val content: String) {
 
     override fun toString(): String {
-        return "Candidate(message='$content')"
+        return "Candidate(content='$content')"
     }
 
     /** List of words in the candidate string */
@@ -115,17 +130,28 @@ class Candidate(val content: String) {
 
     /** P-value that character distribution corresponds to Finnish */
     val monogramPValue: Double by lazy {
-        ChiSquared(FinnishCharacters, thisLetterCounts(), 1.0).pValue()
+        ChiSquared(FinnishCharacters, thisLetterCounts(), 0.1).pValue()
     }
 
     /** P-value that syllable type distribution corresponds to Finnish */
     val syllableTypePValue: Double by lazy {
-        ChiSquared(FinnishSyllables, syllableTypes(), 1.0).pValue()
+        ChiSquared(FinnishSyllables, syllableTypes(), 0.1).pValue()
     }
 
-    val avgPValue: Double by lazy {
-        (monogramPValue + syllableTypePValue) / 2
-    }
+    /**
+     * Declare if candidate is likely Finnish language
+     *
+     * TODO create better statistics
+     */
+    val isLikelyFinnish: Boolean by lazy { monogramPValue > 0.05 }
+
+    /**
+     * Probability of being Finnish language
+     *
+     * TODO create better statistics
+     */
+    val finnishProbability: Double by lazy { monogramPValue }
+
 
     /** Count letters in content */
     private fun thisLetterCounts(): Map<Char, Int> = content.toLowerCase().replace(Characters.notLetter, "")
@@ -147,4 +173,21 @@ class Candidate(val content: String) {
             }
             .groupingBy { it }.eachCount()
     }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as Candidate
+
+        if (content != other.content) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        return content.hashCode()
+    }
+
+
 }

src/main/kotlin/fi/mkouhia/solidabis/bullshit/BullshitConnection.kt

@@ -13,6 +13,20 @@ class BullshitConnection(val bullshits: List<Bullshit>) {
         return "BullshitConnection(bullshits=$bullshits)"
     }
 
+    val notBullshits: List<Bullshit> by lazy {
+        bullshits
+            .filter { it.isLikelyFinnish }
+            .toList()
+            .sortedBy { -it.finnishProbability }
+    }
+
+    val actualBullshits: List<Bullshit> by lazy {
+        bullshits
+            .filter { !it.isLikelyFinnish }
+            .toList()
+            .sortedBy { -it.finnishProbability }
+    }
+
     companion object {
 
         suspend fun fromSecret(secret: Secret, client: HttpClient): BullshitConnection {

src/main/kotlin/fi/mkouhia/solidabis/bullshit/Main.kt

@@ -1,13 +1,10 @@
 package fi.mkouhia.solidabis.bullshit
 
-import fi.mkouhia.solidabis.bullshit.FrequencyDistribution.Companion.FinnishCharacters
 import io.ktor.client.HttpClient
 import io.ktor.client.engine.jetty.Jetty
 import io.ktor.client.features.json.JsonFeature
 import io.ktor.client.features.json.serializer.KotlinxSerializer
 import kotlinx.coroutines.runBlocking
-import java.util.stream.Collectors
-import java.util.stream.IntStream
 
 fun main(args: Array<String>) {
     val secretUrl = "https://koodihaaste-api.solidabis.com/secret"
@@ -22,31 +19,16 @@ fun main(args: Array<String>) {
         }
     }
 
-
-    println(bullshitConnection.bullshits[0])
-    val bs = bullshitConnection.bullshits[0]
-    val msg = bs.message
-    val bs2 = Bullshit(msg)
-
-
-    println(bs2.bestCandidate())
-
-    bullshitConnection.bullshits
-        .map { it.bestCandidate() }
-        .toList()
-        .sortedBy { -it.monogramPValue }
+    println("# No bullshit\n")
+    bullshitConnection.notBullshits
         .forEach {
-            println("${it.avgPValue} ${it.monogramPValue} ${it.syllableTypePValue} ${it.content}")
+            println("%5.1f %% : ${it.bestCandidate.content}".format(100 * it.bestCandidate.finnishProbability))
         }
 
-//    println(bestCandidate(bullshitConnection.bullshits[0].candidates).avgPValue)
-
-    println(FinnishCharacters.frequency)
+    println("\n\n# Bullshit\n")
+    bullshitConnection.actualBullshits
+        .forEach {
+            println("%5.1f %% : ${it.bestCandidate.content}".format(100 * it.bestCandidate.finnishProbability))
+        }
 
-//    val s = bullshitConnection.bullshits
-//        .map {
-//            bestCandidate(it.candidates)
-//        }
-//        .map { it.syllableTypes() }
-//    println(s)
 }

src/main/resources/templates/thymeleaf/index.html

@@ -5,6 +5,31 @@
 <title>Title</title>
 </head>
 <body>
-<span th:text="${user.name}"></span>
+
+<h2>No bullshit</h2>
+<table>
+    <tr>
+        <th>Lause suomeksi</th>
+        <th>Suomenkielisyyden todennäköisyys</th>
+    </tr>
+    <tr th:each="bs : ${bullshitConnection.notBullshits}">
+        <td th:text="${bs.bestCandidate.content}"></td>
+        <td th:text="${#numbers.formatDecimal(100 * bs.bestCandidate.finnishProbability, 1, 1, 'COMMA')}"></td>
+    </tr>
+</table>
+
+
+<h2>Bullshit</h2>
+<table>
+    <tr>
+        <th>Lause alkuperäismuodossaan</th>
+        <th>Suomenkielisyyden todennäköisyys</th>
+    </tr>
+    <tr th:each="bsa : ${bullshitConnection.actualBullshits}">
+        <td th:text="${bsa.message}"></td>
+        <td th:text="${#numbers.formatDecimal(100 * bsa.bestCandidate.finnishProbability, 1, 1, 'COMMA')}"></td>
+    </tr>
+</table>
+
 </body>
 </html>

src/test/kotlin/fi/mkouhia/solidabis/bullshit/ApplicationTest.kt

@@ -1,20 +1,21 @@
 package fi.mkouhia.solidabis.bullshit
 
-import io.kotlintest.specs.AnnotationSpec
+import io.kotlintest.matchers.string.shouldHaveMinLength
+import io.kotlintest.shouldBe
+import io.kotlintest.specs.StringSpec
 import io.ktor.http.HttpMethod
 import io.ktor.http.HttpStatusCode
 import io.ktor.server.testing.handleRequest
 import io.ktor.server.testing.withTestApplication
-import kotlin.test.assertEquals
 
-class ApplicationTest: AnnotationSpec() {
-    @Test
-    fun testRoot() {
+class ApplicationTest: StringSpec({
+
+    "Request at / should return something" {
         withTestApplication({ module() }) {
             handleRequest(HttpMethod.Get, "/").apply {
-                assertEquals(HttpStatusCode.OK, response.status())
-                assertEquals("HELLO WORLD!", response.content)
+                response.status() shouldBe HttpStatusCode.OK
+                response.content shouldHaveMinLength 1
             }
         }
     }
-}
+})

src/test/kotlin/fi/mkouhia/solidabis/bullshit/BullshitTest.kt

@@ -1,6 +1,7 @@
 package fi.mkouhia.solidabis.bullshit
 
 import io.kotlintest.data.forall
+import io.kotlintest.inspectors.forAll
 import io.kotlintest.shouldBe
 import io.kotlintest.specs.StringSpec
 import io.kotlintest.tables.row
@@ -45,4 +46,38 @@ internal class BullshitTest: StringSpec({
         Bullshit("aa").candidates.size shouldBe 29
     }
 
+    "Same content bullshits are equal" {
+        Bullshit("foo") shouldBe Bullshit("foo")
+    }
+
+    "Finnish sentences are recognized as likely Finnish" {
+        listOf(
+            Bullshit("Olen omena, olen pyöreä omena"),
+            Bullshit("Hopeinen kuu luo merelle siltaa, ei tulla koskaan voisi kai tällaista iltaa")
+        ).forAll {
+            it.isLikelyFinnish shouldBe true
+        }
+    }
+
+    "Non-Finnish sentences are recognized as likely not Finnish" {
+        listOf(
+            Bullshit("foo santeohusteoa aosetua oaseta asote ua oaestueoa aoeurga ej."),
+            Bullshit("asd saoe lkar crdcgbok saeu xntbag sacuoa asoueh asoeucbka aoexb")
+        ).forAll {
+            it.isLikelyFinnish shouldBe false
+        }
+    }
+
+    "ROT-13 Finnish sentences are recognized as likely Finnish" {
+        listOf(
+            Bullshit("Olen omena, olen pyöreä omena"),
+            Bullshit("Hopeinen kuu luo merelle siltaa, ei tulla koskaan voisi kai tällaista iltaa")
+        ).map {
+            val msg: String = it.candidates[13].content
+            Bullshit(msg)
+        }.forAll {
+            it.isLikelyFinnish shouldBe true
+        }
+    }
+
 })