Add wiki import

build.gradle

@@ -86,6 +86,7 @@ kotlin {
                 implementation 'org.jetbrains:kotlin-css-jvm:1.0.0-pre.70-kotlin-1.3.21'
 
                 implementation "io.ktor:ktor-html-builder:$ktor_version"
+                implementation "io.ktor:ktor-client-apache:$ktor_version"
 
                 implementation 'org.xerial:sqlite-jdbc:3.25.2'
                 implementation 'org.jetbrains.exposed:exposed:0.12.2'

portal.toml

@@ -3,4 +3,7 @@ host = "localhost"
 port = 8080
 
 [schedule]
-reference = "2019-06-12"
+reference = "2019-06-12"
+
+[general]
+wiki_url = "https://wiki.kif.rocks/w/index.php?title=KIF470:Arbeitskreise&action=raw"

src/commonMain/kotlin/de/kif/common/model/WorkGroup.kt

@@ -7,6 +7,7 @@ import kotlinx.serialization.Serializable
 data class WorkGroup(
     override val id: Long?,
     val name: String,
+    val description: String,
     val interested: Int,
     val track: Track?,
     val projector: Boolean,

src/jvmMain/kotlin/de/kif/backend/Configuration.kt

@@ -73,6 +73,7 @@ object Configuration {
 
     private object GeneralSpec : ConfigSpec("general") {
         val allowedUploadExtensions by required<String>("allowed_upload_extensions")
+        val wikiUrl by required<String>("wiki_url")
     }
 
     object General {
@@ -80,6 +81,7 @@ object Configuration {
         val allowedUploadExtensionSet by lazy {
             allowedUploadExtensions.split(",").map { it.trim().toLowerCase() }.toSet()
         }
+        val wikiUrl by c(GeneralSpec.wikiUrl)
     }
 
     init {

src/jvmMain/kotlin/de/kif/backend/database/Schema.kt

@@ -6,7 +6,7 @@ import org.jetbrains.exposed.sql.Table
 
 object DbTrack : Table() {
     val id = long("id").autoIncrement().primaryKey()
-    val name = varchar("name", 64)
+    val name = varchar("name", 2048)
     val color = varchar("color", 32)
 
     val createdAt = long("createdAt")
@@ -15,7 +15,8 @@ object DbTrack : Table() {
 
 object DbWorkGroup : Table() {
     val id = long("id").autoIncrement().primaryKey()
-    val name = varchar("name", 64)
+    val name = varchar("name", 2048)
+    val description = text("description")
 
     val interested = integer("interested")
     val trackId = long("track_id").nullable()
@@ -38,7 +39,7 @@ object DbWorkGroup : Table() {
 
 object DbRoom : Table() {
     val id = long("id").autoIncrement().primaryKey()
-    val name = varchar("name", 64)
+    val name = varchar("name", 2048)
 
     val places = integer("places")
     val projector = bool("projector")
@@ -79,7 +80,7 @@ object DbUserPermission : Table() {
 
 object DbPost : Table() {
     val id = long("id").autoIncrement().primaryKey()
-    val name = varchar("name", 64)
+    val name = varchar("name", 2048)
 
     val content = text("content")
     val url = varchar("url", 64).uniqueIndex()

src/jvmMain/kotlin/de/kif/backend/repository/WorkGroupRepository.kt

@@ -24,6 +24,7 @@ object WorkGroupRepository : Repository<WorkGroup> {
     private suspend fun rowToModel(row: ResultRow): WorkGroup {
         val id = row[DbWorkGroup.id]
         val name = row[DbWorkGroup.name]
+        val description = row[DbWorkGroup.description]
         val interested = row[DbWorkGroup.interested]
         val trackId = row[DbWorkGroup.trackId]
         val projector = row[DbWorkGroup.projector]
@@ -44,6 +45,7 @@ object WorkGroupRepository : Repository<WorkGroup> {
         return WorkGroup(
             id,
             name,
+            description,
             interested,
             track,
             projector,
@@ -76,6 +78,7 @@ object WorkGroupRepository : Repository<WorkGroup> {
         return dbQuery {
             val id = DbWorkGroup.insert {
                 it[name] = model.name
+                it[description] = model.description
                 it[interested] = model.interested
                 it[trackId] = model.track?.id
                 it[projector] = model.projector
@@ -106,6 +109,7 @@ object WorkGroupRepository : Repository<WorkGroup> {
         dbQuery {
             DbWorkGroup.update({ DbWorkGroup.id eq model.id }) {
                 it[name] = model.name
+                it[description] = model.description
                 it[interested] = model.interested
                 it[trackId] = model.track?.id
                 it[projector] = model.projector

src/jvmMain/kotlin/de/kif/backend/route/Account.kt

@@ -3,7 +3,10 @@ package de.kif.backend.route
 import de.kif.backend.authenticate
 import de.kif.backend.authenticateOrRedirect
 import de.kif.backend.backup.Backup
+import de.kif.backend.repository.TrackRepository
+import de.kif.backend.repository.WorkGroupRepository
 import de.kif.backend.route.api.error
+import de.kif.backend.util.WikiImporter
 import de.kif.backend.view.MainTemplate
 import de.kif.backend.view.MenuTemplate
 import de.kif.common.RepositoryType
@@ -16,16 +19,25 @@ import io.ktor.http.content.PartData
 import io.ktor.http.content.forEachPart
 import io.ktor.http.content.streamProvider
 import io.ktor.request.receiveMultipart
+import io.ktor.request.receiveParameters
 import io.ktor.response.respondRedirect
 import io.ktor.response.respondText
 import io.ktor.routing.Route
 import io.ktor.routing.get
 import io.ktor.routing.post
+import io.ktor.util.toMap
 import kotlinx.html.*
+import mu.KotlinLogging
+
+private val logger = KotlinLogging.logger {}
 
 fun Route.account() {
     get("/account") {
         authenticateOrRedirect { user ->
+
+            val tracks = TrackRepository.all()
+            val wikiSections = WikiImporter.loadSections()
+
             call.respondHtmlTemplate(MainTemplate()) {
                 menuTemplate {
                     this.user = user
@@ -45,7 +57,7 @@ fun Route.account() {
                         }
                     }
 
-                    div {
+                    div("account-backup") {
                         if (user.checkPermission(Permission.ROOM)) {
                             a("/account/backup/rooms.json", classes = "form-btn") {
                                 attributes["download"] = "rooms-backup"
@@ -88,48 +100,115 @@ fun Route.account() {
                             }
                         }
                     }
-                    div {
-                        form(
-                            action = "/account/import",
-                            method = FormMethod.post,
-                            encType = FormEncType.multipartFormData
-                        ) {
-                            div("form-group") {
-                                label {
-                                    htmlFor = "backup"
-                                    +"Backup image"
-                                }
-                                input(
-                                    name = "backup",
-                                    classes = "form-btn",
-                                    type = InputType.file
-                                ) {
-                                    id = "backup"
-                                    value = "Select backup image"
-                                    accept = ".json"
-                                }
-                            }
 
-                            div("form-switch-group") {
+                    if (user.checkPermission(Permission.ADMIN)) {
-                                div("form-group form-switch") {
+                        div("account-import") {
-                                    input(
+                            form(
-                                        name = "reset",
+                                action = "/account/import",
-                                        classes = "form-control",
+                                method = FormMethod.post,
-                                        type = InputType.checkBox
+                                encType = FormEncType.multipartFormData
-                                    ) {
+                            ) {
-                                        id = "reset"
+                                div("form-group") {
-                                        checked = false
-                                    }
                                     label {
-                                        htmlFor = "reset"
+                                        htmlFor = "backup"
-                                        +"Reset"
+                                        +"Backup image"
+                                    }
+                                    input(
+                                        name = "backup",
+                                        classes = "form-btn",
+                                        type = InputType.file
+                                    ) {
+                                        id = "backup"
+                                        value = "Select backup image"
+                                        accept = ".json"
+                                    }
+                                }
+
+                                div("form-switch-group") {
+                                    div("form-group form-switch") {
+                                        input(
+                                            name = "reset",
+                                            classes = "form-control",
+                                            type = InputType.checkBox
+                                        ) {
+                                            id = "reset"
+                                            checked = false
+                                        }
+                                        label {
+                                            htmlFor = "reset"
+                                            +"Reset"
+                                        }
+                                    }
+                                }
+
+                                div("form-group") {
+                                    button(type = ButtonType.submit, classes = "form-btn btn-primary") {
+                                        +"Import"
                                     }
                                 }
                             }
+                        }
+                    }
+
+                    if (user.checkPermission(Permission.ADMIN)) {
+                        div("account-import-wiki") {
+                            span { +"Import work group data from the kif wiki" }
+
+                            form(action = "/account/import-wiki", method = FormMethod.post) {
+                                for ((index, section) in wikiSections.withIndex()) {
+                                    div("form-group") {
+                                        label {
+                                            htmlFor = "section-$index-track"
+                                            +section
+                                        }
+                                        select(
+                                            classes = "form-control"
+                                        ) {
+                                            name = "section-$index-track"
+
+                                            option {
+                                                selected = false
+                                                value = "-1"
+                                                +"Do not import"
+                                            }
+                                            option {
+                                                selected = true
+                                                value = "null"
+                                                +"None"
+                                            }
+                                            for (track in tracks) {
+                                                option {
+                                                    selected = false
+                                                    value = track.id.toString()
+                                                    +track.name
+                                                }
+                                            }
+                                        }
+                                        input(type = InputType.hidden, name = "section-$index-name") {
+                                            value = section
+                                        }
+                                    }
+                                }
+
+                                div("form-switch-group") {
+                                    div("form-group form-switch") {
+                                        input(
+                                            name = "skip-existing",
+                                            classes = "form-control",
+                                            type = InputType.checkBox
+                                        ) {
+                                            id = "skip-existing"
+                                            checked = false
+                                        }
+                                        label {
+                                            htmlFor = "skip-existing"
+                                            +"Skip existing work groups"
+                                        }
+                                    }
+                                }
 
-                            div("form-group") {
                                 button(type = ButtonType.submit, classes = "form-btn btn-primary") {
-                                    +"Import"
+                                    +"Import wiki"
                                 }
                             }
                         }
@@ -188,7 +267,7 @@ fun Route.account() {
     }
 
     post("/account/import") {
-        authenticateOrRedirect(Permission.ADMIN) {
+        authenticate(Permission.ADMIN) {
             var reset = false
             var import = ""
 
@@ -213,6 +292,54 @@ fun Route.account() {
             }
 
             call.respondRedirect("/account")
+        } onFailure {
+            call.error(HttpStatusCode.Unauthorized)
+        }
+    }
+
+    post("/account/import-wiki") {
+        authenticate(Permission.ADMIN) {
+            val params = call.receiveParameters().toMap().mapValues { (_, list) ->
+                list.firstOrNull()
+            }
+
+            val skipExisting = params["skip-existing"] == "on"
+
+            val map = mutableMapOf<Int, Pair<String, Long?>>()
+
+            for ((key, value) in params) {
+                if (key.startsWith("section") && value != null) {
+                    val numberedType = key.substringAfter("-")
+                    val number = numberedType.substringBefore("-").toIntOrNull() ?: continue
+                    val type = numberedType.substringAfter("-")
+                    var pair = map.getOrElse(number) { "" to null }
+                    if (type == "name") {
+                        pair = value to pair.second
+                    } else if (type == "track") {
+                        pair = pair.first to value.toLongOrNull()
+                    }
+                    map[number] = pair
+                }
+            }
+
+            val sections = map.values.toMap()
+
+            val existingWorkGroups = WorkGroupRepository.all().map { it.name }.toSet()
+            val importedWorkGroups = WikiImporter.import(sections = sections)
+
+            var counter = 0
+            for (workGroup in importedWorkGroups) {
+                if (skipExisting && workGroup.name in existingWorkGroups) continue
+
+                WorkGroupRepository.create(workGroup)
+                counter++
+            }
+
+            logger.info { "Import $counter from ${importedWorkGroups.size} work groups!" }
+
+            call.respondRedirect("/account")
+        } onFailure {
+            call.error(HttpStatusCode.Unauthorized)
         }
     }
 }

src/jvmMain/kotlin/de/kif/backend/route/WorkGroup.kt

@@ -186,6 +186,18 @@ fun Route.workGroup() {
                                 value = editWorkGroup.name
                             }
                         }
+                        div("form-group") {
+                            label {
+                                htmlFor = "description"
+                                +"Description"
+                            }
+                            textArea(rows = "10", classes = "form-control") {
+                                name = "description"
+                                id = "description"
+
+                                +editWorkGroup.description
+                            }
+                        }
                         div("form-group") {
                             label {
                                 htmlFor = "interested"
@@ -467,6 +479,7 @@ fun Route.workGroup() {
             var editWorkGroup = WorkGroupRepository.get(workGroupId) ?: return@post
 
             params["name"]?.let { editWorkGroup = editWorkGroup.copy(name = it) }
+            params["description"]?.let { editWorkGroup = editWorkGroup.copy(description = it) }
             params["interested"]?.toIntOrNull()?.let { editWorkGroup = editWorkGroup.copy(interested = it) }
             params["track"]?.toLongOrNull()?.let {
                 val track = TrackRepository.get(it)
@@ -536,6 +549,18 @@ fun Route.workGroup() {
                                 value = ""
                             }
                         }
+                        div("form-group") {
+                            label {
+                                htmlFor = "description"
+                                +"Description"
+                            }
+                            textArea(rows = "10", classes = "form-control") {
+                                name = "description"
+                                id = "description"
+
+                                +""
+                            }
+                        }
                         div("form-group") {
                             label {
                                 htmlFor = "interested"
@@ -740,6 +765,7 @@ fun Route.workGroup() {
             }
 
             val name = params["name"] ?: return@post
+            val description = params["description"] ?: return@post
             val interested = (params["interested"] ?: return@post).toIntOrNull() ?: 0
             val track = (params["track"] ?: return@post).toLongOrNull()?.let { TrackRepository.get(it) }
             val projector = params["projector"] == "on"
@@ -775,6 +801,7 @@ fun Route.workGroup() {
             val workGroup = WorkGroup(
                 null,
                 name = name,
+                description = description,
                 interested = interested,
                 track = track,
                 projector = projector,

src/jvmMain/kotlin/de/kif/backend/util/WikiImporter.kt

@@ -0,0 +1,161 @@
+package de.kif.backend.util
+
+import de.kif.backend.Configuration
+import de.kif.backend.repository.TrackRepository
+import de.kif.common.model.Language
+import de.kif.common.model.Track
+import de.kif.common.model.WorkGroup
+import io.ktor.client.HttpClient
+import io.ktor.client.request.get
+import mu.KotlinLogging
+import kotlin.math.max
+
+object WikiImporter {
+
+    private val logger = KotlinLogging.logger {}
+
+    private suspend fun loadDocument(url: String): String = HttpClient().get(url)
+
+    suspend fun loadSections(url: String = Configuration.General.wikiUrl): List<String> {
+        val data = loadDocument(url)
+
+        val lines = data.split("\n")
+
+        return lines.filter {
+            it.matches("==.*==".toRegex())
+        }.map {
+            it.replace("==", "").trim()
+        }
+    }
+
+    /**
+     * Import ak data from the wiki
+     *
+     * @param url Url to the wiki entry. It must point to the raw version not the rendered ("https://wiki.kif.rocks/w/index.php?title=KIF470:Arbeitskreise&action=raw").
+     * @param sections Track association. A negative id skip's a section. All other id values will be mapped to the
+     * corresponding track or null.
+     */
+    suspend fun import(
+        url: String = Configuration.General.wikiUrl,
+        sections: Map<String, Long?> = emptyMap()
+    ): List<WorkGroup> {
+        logger.info { "Start import of work groups..." }
+
+        val data = loadDocument(url)
+
+        val tracks = TrackRepository.all().associateBy { it.id }
+
+        val workGroups = mutableListOf<WorkGroup>()
+
+        val dataRegex = """(== (.*) ==)|(\{\{.*\n((\|.*\n)+)}})""".toRegex()
+
+        var track: Track? = null
+        var skip = false
+
+        for (dataMatch in dataRegex.findAll(data)) {
+            val header = dataMatch.groupValues[2]
+            val content = dataMatch.groupValues[4]
+
+            if (header.isNotBlank()) {
+                val trackId = sections[header.trim()]
+
+                when {
+                    trackId == null -> {
+                        skip = false
+                        track = null
+                    }
+                    trackId < 0 -> {
+                        skip = true
+                        track = null
+                    }
+                    else -> {
+                        skip = false
+                        track = tracks[trackId]
+                    }
+                }
+            } else if (content.isNotBlank()) {
+                if (skip) continue
+                val ak = content
+                    .split("\n|")
+                    .mapNotNull {
+                        it.split("=")
+                            .map { it.removePrefix("|").trim() }
+                            .let { if (it.size > 1) it[0] to it[1] else null }
+                    }
+                    .toMap()
+
+                println(ak)
+
+                val name = ak["name"]?.trim() ?: continue
+                var desc = ak["beschreibung"]?.trim() ?: ""
+                val participant = ak["wieviele"]?.trim() ?: ""
+                val who = ak["wer"]?.trim() ?: ""
+                val time = ak["wann"]?.trim() ?: ""
+                val length = ak["dauer"]?.trim() ?: ""
+
+                if (name.isBlank() && desc.isBlank()) continue
+
+                if (who.isNotBlank() || time.isNotBlank() || length.isNotBlank()) {
+                    desc += "\n"
+                    desc += "--- Aus dem Wiki übernommen ---"
+                    desc += "\n"
+
+                    if (who.isNotBlank()) {
+                        desc += "Wer = $who\n"
+                    }
+                    if (time.isNotBlank()) {
+                        desc += "Wann = $time\n"
+                    }
+                    if (length.isNotBlank()) {
+                        desc += "Dauer = $length\n"
+                    }
+                }
+
+                var akLength = 60
+
+                if (length.isNotBlank()) {
+                    val regex = """(\d+) *(h|[Ss]tund|[Ss]lot)""".toRegex()
+                    val match = regex.find(length)
+
+                    if (match != null) {
+                        akLength = max(akLength, match.groupValues.getOrNull(1)?.toIntOrNull() ?: 0)
+                    }
+                }
+
+                var interested = 0
+
+                if (length.isNotBlank()) {
+                    val regex = """\d+""".toRegex()
+                    val match = regex.find(participant)
+
+                    if (match != null) {
+                        interested = match.groupValues.getOrNull(0)?.toIntOrNull() ?: 0
+                    }
+                }
+
+                println(1)
+
+                workGroups += WorkGroup(
+                    id = null,
+                    name = name,
+                    description = desc,
+                    interested = interested,
+                    track = track,
+                    projector = false,
+                    resolution = false,
+                    internet = false,
+                    whiteboard = false,
+                    blackboard = false,
+                    accessible = false,
+                    length = akLength,
+                    language = Language.GERMAN,
+                    constraints = emptyList()
+                )
+            }
+        }
+
+        logger.info { "Found ${workGroups.size} work groups" }
+
+        return workGroups
+    }
+}

src/jvmMain/resources/portal.toml

@@ -16,4 +16,5 @@ session_name = "SESSION"
 sign_key = "d1 20 23 8c 01 f8 f0 0d 9d 7c ff 68 21 97 75 31 38 3f fb 91 20 3a 8d 86 d4 e9 d8 50 f8 71 f1 dc"
 
 [general]
-allowed_upload_extensions = "png, jpg, jpeg"
+allowed_upload_extensions = "png, jpg, jpeg"
+wiki_url = ""