Skip to content

Commit

Permalink
Reorganize PostgreSQL code (#541)
Browse files Browse the repository at this point in the history
* Reorganize Postgres configuration

* Create Hikari data source as resource

* Remove duplicated code

* Remove PostgresVectorStoreConfig class

* Remove unnecessary println
  • Loading branch information
franciscodr authored Nov 15, 2023
1 parent dbe5ac1 commit 812b744
Show file tree
Hide file tree
Showing 13 changed files with 183 additions and 296 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package com.xebia.functional.xef.store.config

import kotlinx.serialization.Serializable

@Serializable
data class PostgreSQLVectorStoreConfig(
val url: String,
val driver: String,
val user: String,
val password: String,
val collectionName: String,
val vectorSize: Int,
val migrationsTable: String = "migrations",
val migrationsLocations: List<String> = listOf("vectorStore/migrations")
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package com.xebia.functional.xef.store.migrations

import com.xebia.functional.xef.store.config.PostgreSQLVectorStoreConfig
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.flywaydb.core.Flyway
import org.flywaydb.core.api.configuration.FluentConfiguration
import org.flywaydb.core.api.output.MigrateResult
import javax.sql.DataSource

suspend fun runDatabaseMigrations(
dataSource: DataSource,
migrationsTable: String,
migrationsLocations: List<String>
): MigrateResult =
withContext(Dispatchers.IO) {
val migration: FluentConfiguration = Flyway.configure()
.dataSource(dataSource)
.table(migrationsTable)
.locations(*migrationsLocations.toTypedArray())
.loggers("slf4j")
val isValid = migration.ignoreMigrationPatterns("*:pending").load().validateWithResult()
if (!isValid.validationSuccessful) {
throw IllegalStateException("Migration validation failed: ${isValid.errorDetails}")
}
migration.load().migrate()
}

suspend fun runDatabaseMigrations(
config: PostgreSQLVectorStoreConfig
): MigrateResult =
withContext(Dispatchers.IO) {
with(config) {
val migration: FluentConfiguration = Flyway.configure()
.dataSource(
url,
user,
password
)
.table(migrationsTable)
.locations(*migrationsLocations.toTypedArray())
.loggers("slf4j")
val isValid = migration.ignoreMigrationPatterns("*:pending").load().validateWithResult()
if (!isValid.validationSuccessful) {
throw IllegalStateException("Migration validation failed: ${isValid.errorDetails}")
}
migration.load().migrate()
}
}

This file was deleted.

63 changes: 32 additions & 31 deletions server/src/main/kotlin/com/xebia/functional/xef/server/Server.kt
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,30 @@ import arrow.continuations.SuspendApp
import arrow.continuations.ktor.server
import arrow.fx.coroutines.resourceScope
import com.typesafe.config.ConfigFactory
import com.xebia.functional.xef.server.db.psql.Migrate
import com.xebia.functional.xef.server.db.psql.XefDatabaseConfig
import com.xebia.functional.xef.server.exceptions.exceptionsHandler
import com.xebia.functional.xef.server.http.routes.*
import com.xebia.functional.xef.server.services.PostgresVectorStoreService
import com.xebia.functional.xef.server.services.RepositoryService
import com.xebia.functional.xef.server.services.VectorStoreService
import io.ktor.client.*
import io.ktor.client.engine.cio.*
import io.ktor.client.plugins.auth.*
import com.xebia.functional.xef.server.http.routes.aiRoutes
import com.xebia.functional.xef.server.http.routes.xefRoutes
import com.xebia.functional.xef.server.services.hikariDataSource
import com.xebia.functional.xef.server.services.vectorStoreService
import com.xebia.functional.xef.store.migrations.runDatabaseMigrations
import io.ktor.client.HttpClient
import io.ktor.client.engine.cio.CIO
import io.ktor.client.plugins.auth.Auth
import io.ktor.client.plugins.contentnegotiation.ContentNegotiation as ClientContentNegotiation
import io.ktor.client.plugins.logging.*
import io.ktor.http.*
import io.ktor.serialization.kotlinx.json.*
import io.ktor.server.application.*
import io.ktor.server.auth.*
import io.ktor.server.netty.*
import io.ktor.server.plugins.contentnegotiation.*
import io.ktor.server.plugins.cors.routing.*
import io.ktor.server.resources.*
import io.ktor.server.routing.*
import io.ktor.client.plugins.logging.LogLevel
import io.ktor.client.plugins.logging.Logging
import io.ktor.http.HttpMethod
import io.ktor.serialization.kotlinx.json.json
import io.ktor.server.application.install
import io.ktor.server.auth.Authentication
import io.ktor.server.auth.UserIdPrincipal
import io.ktor.server.auth.bearer
import io.ktor.server.netty.Netty
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
import io.ktor.server.plugins.cors.routing.CORS
import io.ktor.server.resources.Resources
import io.ktor.server.routing.routing
import kotlinx.coroutines.awaitCancellation
import org.jetbrains.exposed.sql.Database
import org.slf4j.LoggerFactory
Expand All @@ -33,24 +36,22 @@ object Server {
@JvmStatic
fun main(args: Array<String>) = SuspendApp {
resourceScope {
val logger = LoggerFactory.getLogger("xef-server")

val config = ConfigFactory.load("database.conf").resolve()
val xefDBConfig = XefDatabaseConfig.load("xef", config)
Migrate.migrate(xefDBConfig)
val xefDBConfig = XefDatabaseConfig.load("xef-database", config)

val logger = LoggerFactory.getLogger("xef-server")
val xefDatasource = hikariDataSource(xefDBConfig.url, xefDBConfig.user, xefDBConfig.password)

val hikariDataSourceXefDB =
RepositoryService.getHikariDataSource(
xefDBConfig.getUrl(),
xefDBConfig.user,
xefDBConfig.password
)
Database.connect(hikariDataSourceXefDB)
runDatabaseMigrations(
xefDatasource,
xefDBConfig.migrationsTable,
xefDBConfig.migrationsLocations
)

val vectorStoreService =
VectorStoreService.load("xef-vector-store", config).getVectorStoreService(logger)
Database.connect(xefDatasource)

(vectorStoreService as? PostgresVectorStoreService)?.addCollection()
vectorStoreService("xef-vector-store", config, logger)

val ktorClient =
HttpClient(CIO) {
Expand Down

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,33 +1,26 @@
package com.xebia.functional.xef.server.db.psql

import com.typesafe.config.Config
import com.typesafe.config.ConfigFactory
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.Serializable
import kotlinx.serialization.hocon.Hocon

@Serializable
class XefDatabaseConfig(
val host: String,
val port: Int,
val database: String,
data class XefDatabaseConfig(
val url: String,
val user: String,
val password: String,
val migrationsTable: String,
val migrationsLocations: List<String>
) {

fun getUrl(): String = "jdbc:postgresql://$host:$port/$database"

companion object {
@OptIn(ExperimentalSerializationApi::class)
suspend fun load(configNamespace: String, config: Config? = null): XefDatabaseConfig =
suspend fun load(configNamespace: String, config: Config): XefDatabaseConfig =
withContext(Dispatchers.IO) {
val rawConfig = config ?: ConfigFactory.load().resolve()
val jdbcConfig = rawConfig.getConfig(configNamespace)
Hocon.decodeFromConfig(serializer(), jdbcConfig)
val databaseConfig = config.getConfig(configNamespace)
Hocon.decodeFromConfig(serializer(), databaseConfig)
}
}
}
Loading

0 comments on commit 812b744

Please sign in to comment.