kif/portal
10
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add wiki import

Browse source
This commit is contained in:
Lars Westermann 2019-05-31 11:40:17 +02:00
parent c6620d3395
commit 5c058f7fdc
Signed by: lars.westermann
GPG key ID: 9D417FA5BB9D5E1D
10 changed files with 371 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
build.gradle
View file

@ -86,6 +86,7 @@ kotlin {
implementation 'org.jetbrains:kotlin-css-jvm:1.0.0-pre.70-kotlin-1.3.21'
implementation "io.ktor:ktor-html-builder:$ktor_version"
implementation "io.ktor:ktor-client-apache:$ktor_version"
implementation 'org.xerial:sqlite-jdbc:3.25.2'
implementation 'org.jetbrains.exposed:exposed:0.12.2'

3
portal.toml
View file

@ -4,3 +4,6 @@ port = 8080
[schedule]
reference = "2019-06-12"
[general]
wiki_url = "https://wiki.kif.rocks/w/index.php?title=KIF470:Arbeitskreise&action=raw"

1
src/commonMain/kotlin/de/kif/common/model/WorkGroup.kt
View file

@ -7,6 +7,7 @@ import kotlinx.serialization.Serializable
data class WorkGroup(
override val id: Long?,
val name: String,
val description: String,
val interested: Int,
val track: Track?,
val projector: Boolean,

2
src/jvmMain/kotlin/de/kif/backend/Configuration.kt
View file

@ -73,6 +73,7 @@ object Configuration {
private object GeneralSpec : ConfigSpec("general") {
val allowedUploadExtensions by required<String>("allowed_upload_extensions")
val wikiUrl by required<String>("wiki_url")
}
object General {
@ -80,6 +81,7 @@ object Configuration {
val allowedUploadExtensionSet by lazy {
allowedUploadExtensions.split(",").map { it.trim().toLowerCase() }.toSet()
}
val wikiUrl by c(GeneralSpec.wikiUrl)
}
init {

9
src/jvmMain/kotlin/de/kif/backend/database/Schema.kt
View file

@ -6,7 +6,7 @@ import org.jetbrains.exposed.sql.Table
object DbTrack : Table() {
val id = long("id").autoIncrement().primaryKey()
val name = varchar("name", 64)
val name = varchar("name", 2048)
val color = varchar("color", 32)
val createdAt = long("createdAt")
@ -15,7 +15,8 @@ object DbTrack : Table() {
object DbWorkGroup : Table() {
val id = long("id").autoIncrement().primaryKey()
val name = varchar("name", 64)
val name = varchar("name", 2048)
val description = text("description")
val interested = integer("interested")
val trackId = long("track_id").nullable()
@ -38,7 +39,7 @@ object DbWorkGroup : Table() {
object DbRoom : Table() {
val id = long("id").autoIncrement().primaryKey()
val name = varchar("name", 64)
val name = varchar("name", 2048)
val places = integer("places")
val projector = bool("projector")
@ -79,7 +80,7 @@ object DbUserPermission : Table() {
object DbPost : Table() {
val id = long("id").autoIncrement().primaryKey()
val name = varchar("name", 64)
val name = varchar("name", 2048)
val content = text("content")
val url = varchar("url", 64).uniqueIndex()

4
src/jvmMain/kotlin/de/kif/backend/repository/WorkGroupRepository.kt
View file

@ -24,6 +24,7 @@ object WorkGroupRepository : Repository<WorkGroup> {
private suspend fun rowToModel(row: ResultRow): WorkGroup {
val id = row[DbWorkGroup.id]
val name = row[DbWorkGroup.name]
val description = row[DbWorkGroup.description]
val interested = row[DbWorkGroup.interested]
val trackId = row[DbWorkGroup.trackId]
val projector = row[DbWorkGroup.projector]
@ -44,6 +45,7 @@ object WorkGroupRepository : Repository<WorkGroup> {
return WorkGroup(
id,
name,
description,
interested,
track,
projector,
@ -76,6 +78,7 @@ object WorkGroupRepository : Repository<WorkGroup> {
return dbQuery {
val id = DbWorkGroup.insert {
it[name] = model.name
it[description] = model.description
it[interested] = model.interested
it[trackId] = model.track?.id
it[projector] = model.projector
@ -106,6 +109,7 @@ object WorkGroupRepository : Repository<WorkGroup> {
dbQuery {
DbWorkGroup.update({ DbWorkGroup.id eq model.id }) {
it[name] = model.name
it[description] = model.description
it[interested] = model.interested
it[trackId] = model.track?.id
it[projector] = model.projector

133
src/jvmMain/kotlin/de/kif/backend/route/Account.kt
View file

@ -3,7 +3,10 @@ package de.kif.backend.route
import de.kif.backend.authenticate
import de.kif.backend.authenticateOrRedirect
import de.kif.backend.backup.Backup
import de.kif.backend.repository.TrackRepository
import de.kif.backend.repository.WorkGroupRepository
import de.kif.backend.route.api.error
import de.kif.backend.util.WikiImporter
import de.kif.backend.view.MainTemplate
import de.kif.backend.view.MenuTemplate
import de.kif.common.RepositoryType
@ -16,16 +19,25 @@ import io.ktor.http.content.PartData
import io.ktor.http.content.forEachPart
import io.ktor.http.content.streamProvider
import io.ktor.request.receiveMultipart
import io.ktor.request.receiveParameters
import io.ktor.response.respondRedirect
import io.ktor.response.respondText
import io.ktor.routing.Route
import io.ktor.routing.get
import io.ktor.routing.post
import io.ktor.util.toMap
import kotlinx.html.*
import mu.KotlinLogging
private val logger = KotlinLogging.logger {}
fun Route.account() {
get("/account") {
authenticateOrRedirect { user ->
val tracks = TrackRepository.all()
val wikiSections = WikiImporter.loadSections()
call.respondHtmlTemplate(MainTemplate()) {
menuTemplate {
this.user = user
@ -45,7 +57,7 @@ fun Route.account() {
}
}
div {
div("account-backup") {
if (user.checkPermission(Permission.ROOM)) {
a("/account/backup/rooms.json", classes = "form-btn") {
attributes["download"] = "rooms-backup"
@ -88,7 +100,9 @@ fun Route.account() {
}
}
}
div {
if (user.checkPermission(Permission.ADMIN)) {
div("account-import") {
form(
action = "/account/import",
method = FormMethod.post,
@ -135,6 +149,71 @@ fun Route.account() {
}
}
}
if (user.checkPermission(Permission.ADMIN)) {
div("account-import-wiki") {
span { +"Import work group data from the kif wiki" }
form(action = "/account/import-wiki", method = FormMethod.post) {
for ((index, section) in wikiSections.withIndex()) {
div("form-group") {
label {
htmlFor = "section-$index-track"
+section
}
select(
classes = "form-control"
) {
name = "section-$index-track"
option {
selected = false
value = "-1"
+"Do not import"
}
option {
selected = true
value = "null"
+"None"
}
for (track in tracks) {
option {
selected = false
value = track.id.toString()
+track.name
}
}
}
input(type = InputType.hidden, name = "section-$index-name") {
value = section
}
}
}
div("form-switch-group") {
div("form-group form-switch") {
input(
name = "skip-existing",
classes = "form-control",
type = InputType.checkBox
) {
id = "skip-existing"
checked = false
}
label {
htmlFor = "skip-existing"
+"Skip existing work groups"
}
}
}
button(type = ButtonType.submit, classes = "form-btn btn-primary") {
+"Import wiki"
}
}
}
}
}
}
}
}
@ -188,7 +267,7 @@ fun Route.account() {
}
post("/account/import") {
authenticateOrRedirect(Permission.ADMIN) {
authenticate(Permission.ADMIN) {
var reset = false
var import = ""
@ -213,6 +292,54 @@ fun Route.account() {
}
call.respondRedirect("/account")
} onFailure {
call.error(HttpStatusCode.Unauthorized)
}
}
post("/account/import-wiki") {
authenticate(Permission.ADMIN) {
val params = call.receiveParameters().toMap().mapValues { (_, list) ->
list.firstOrNull()
}
val skipExisting = params["skip-existing"] == "on"
val map = mutableMapOf<Int, Pair<String, Long?>>()
for ((key, value) in params) {
if (key.startsWith("section") && value != null) {
val numberedType = key.substringAfter("-")
val number = numberedType.substringBefore("-").toIntOrNull() ?: continue
val type = numberedType.substringAfter("-")
var pair = map.getOrElse(number) { "" to null }
if (type == "name") {
pair = value to pair.second
} else if (type == "track") {
pair = pair.first to value.toLongOrNull()
}
map[number] = pair
}
}
val sections = map.values.toMap()
val existingWorkGroups = WorkGroupRepository.all().map { it.name }.toSet()
val importedWorkGroups = WikiImporter.import(sections = sections)
var counter = 0
for (workGroup in importedWorkGroups) {
if (skipExisting && workGroup.name in existingWorkGroups) continue
WorkGroupRepository.create(workGroup)
counter++
}
logger.info { "Import $counter from ${importedWorkGroups.size} work groups!" }
call.respondRedirect("/account")
} onFailure {
call.error(HttpStatusCode.Unauthorized)
}
}
}

27
src/jvmMain/kotlin/de/kif/backend/route/WorkGroup.kt
View file

@ -186,6 +186,18 @@ fun Route.workGroup() {
value = editWorkGroup.name
}
}
div("form-group") {
label {
htmlFor = "description"
+"Description"
}
textArea(rows = "10", classes = "form-control") {
name = "description"
id = "description"
+editWorkGroup.description
}
}
div("form-group") {
label {
htmlFor = "interested"
@ -467,6 +479,7 @@ fun Route.workGroup() {
var editWorkGroup = WorkGroupRepository.get(workGroupId) ?: return@post
params["name"]?.let { editWorkGroup = editWorkGroup.copy(name = it) }
params["description"]?.let { editWorkGroup = editWorkGroup.copy(description = it) }
params["interested"]?.toIntOrNull()?.let { editWorkGroup = editWorkGroup.copy(interested = it) }
params["track"]?.toLongOrNull()?.let {
val track = TrackRepository.get(it)
@ -536,6 +549,18 @@ fun Route.workGroup() {
value = ""
}
}
div("form-group") {
label {
htmlFor = "description"
+"Description"
}
textArea(rows = "10", classes = "form-control") {
name = "description"
id = "description"
+""
}
}
div("form-group") {
label {
htmlFor = "interested"
@ -740,6 +765,7 @@ fun Route.workGroup() {
}
val name = params["name"] ?: return@post
val description = params["description"] ?: return@post
val interested = (params["interested"] ?: return@post).toIntOrNull() ?: 0
val track = (params["track"] ?: return@post).toLongOrNull()?.let { TrackRepository.get(it) }
val projector = params["projector"] == "on"
@ -775,6 +801,7 @@ fun Route.workGroup() {
val workGroup = WorkGroup(
null,
name = name,
description = description,
interested = interested,
track = track,
projector = projector,

161
src/jvmMain/kotlin/de/kif/backend/util/WikiImporter.kt Normal file
View file

@ -0,0 +1,161 @@
package de.kif.backend.util
import de.kif.backend.Configuration
import de.kif.backend.repository.TrackRepository
import de.kif.common.model.Language
import de.kif.common.model.Track
import de.kif.common.model.WorkGroup
import io.ktor.client.HttpClient
import io.ktor.client.request.get
import mu.KotlinLogging
import kotlin.math.max
object WikiImporter {
private val logger = KotlinLogging.logger {}
private suspend fun loadDocument(url: String): String = HttpClient().get(url)
suspend fun loadSections(url: String = Configuration.General.wikiUrl): List<String> {
val data = loadDocument(url)
val lines = data.split("\n")
return lines.filter {
it.matches("==.*==".toRegex())
}.map {
it.replace("==", "").trim()
}
}
/**
* Import ak data from the wiki
*
* @param url Url to the wiki entry. It must point to the raw version not the rendered ("https://wiki.kif.rocks/w/index.php?title=KIF470:Arbeitskreise&action=raw").
* @param sections Track association. A negative id skip's a section. All other id values will be mapped to the
* corresponding track or null.
*/
suspend fun import(
url: String = Configuration.General.wikiUrl,
sections: Map<String, Long?> = emptyMap()
): List<WorkGroup> {
logger.info { "Start import of work groups..." }
val data = loadDocument(url)
val tracks = TrackRepository.all().associateBy { it.id }
val workGroups = mutableListOf<WorkGroup>()
val dataRegex = """(== (.*) ==)|(\{\{.*\n((\|.*\n)+)}})""".toRegex()
var track: Track? = null
var skip = false
for (dataMatch in dataRegex.findAll(data)) {
val header = dataMatch.groupValues[2]
val content = dataMatch.groupValues[4]
if (header.isNotBlank()) {
val trackId = sections[header.trim()]
when {
trackId == null -> {
skip = false
track = null
}
trackId < 0 -> {
skip = true
track = null
}
else -> {
skip = false
track = tracks[trackId]
}
}
} else if (content.isNotBlank()) {
if (skip) continue
val ak = content
.split("\n|")
.mapNotNull {
it.split("=")
.map { it.removePrefix("|").trim() }
.let { if (it.size > 1) it[0] to it[1] else null }
}
.toMap()
println(ak)
val name = ak["name"]?.trim() ?: continue
var desc = ak["beschreibung"]?.trim() ?: ""
val participant = ak["wieviele"]?.trim() ?: ""
val who = ak["wer"]?.trim() ?: ""
val time = ak["wann"]?.trim() ?: ""
val length = ak["dauer"]?.trim() ?: ""
if (name.isBlank() && desc.isBlank()) continue
if (who.isNotBlank() || time.isNotBlank() || length.isNotBlank()) {
desc += "\n"
desc += "--- Aus dem Wiki übernommen ---"
desc += "\n"
if (who.isNotBlank()) {
desc += "Wer = $who\n"
}
if (time.isNotBlank()) {
desc += "Wann = $time\n"
}
if (length.isNotBlank()) {
desc += "Dauer = $length\n"
}
}
var akLength = 60
if (length.isNotBlank()) {
val regex = """(\d+) *(h|[Ss]tund|[Ss]lot)""".toRegex()
val match = regex.find(length)
if (match != null) {
akLength = max(akLength, match.groupValues.getOrNull(1)?.toIntOrNull() ?: 0)
}
}
var interested = 0
if (length.isNotBlank()) {
val regex = """\d+""".toRegex()
val match = regex.find(participant)
if (match != null) {
interested = match.groupValues.getOrNull(0)?.toIntOrNull() ?: 0
}
}
println(1)
workGroups += WorkGroup(
id = null,
name = name,
description = desc,
interested = interested,
track = track,
projector = false,
resolution = false,
internet = false,
whiteboard = false,
blackboard = false,
accessible = false,
length = akLength,
language = Language.GERMAN,
constraints = emptyList()
)
}
}
logger.info { "Found ${workGroups.size} work groups" }
return workGroups
}
}

1
src/jvmMain/resources/portal.toml
View file

@ -17,3 +17,4 @@ sign_key = "d1 20 23 8c 01 f8 f0 0d 9d 7c ff 68 21 97 75 31 38 3f fb 91 20 3a 8d
[general]
allowed_upload_extensions = "png, jpg, jpeg"
wiki_url = ""