Database setup

README.md

@@ -39,14 +39,21 @@ Visit [localhost:1234](http://localhost:1234/) and you should see the frontend.
 
 Code is in `src/main/{css,html,scala}`
 
+### Mar 7, 2024
 
-### Next time
- - Set up a DB with postgres / skunk with a single table for comments.  
- - Goal: Have a comment box displayed on screen
- - Goodreads style.
- - Get a book with a given id
-    - Filter books by fields?
-    - Have those fields as query parameters?
+
+
+### Feb 22, 2024
+
+- We attempted to setup a database with a table to store book. We decided to run this all as tasks from sbt using
+  - Docker to create the database
+  - Flyway to populate the database
+
+- Errors to resolve:
+  - snorri": -c: line 1: unexpected EOF while looking for matching `"'
+  - how do we specify flyway files location to Flyway.configure
+
+- We spent a lot of time doing devops style work. Not very much programming.
 
 ### Feb 8, 2024
   - Set a target: We're going all into web development! Plus DBs. 

backend/src/main/resources/book.json

@@ -1,10 +1,12 @@
 [
   {
-    "id": "978-0641723445",
+    "id": 100,
+    "isbn": "978-0641723445",
     "name": "The Lightning Thief",
     "author": "Rick Riordan",
-    "price": 12.50,
     "pages": 384,
+    "publishedYear": 2002,
+    "price": 12.50,
     "genres": "fantasy",
     "inStock": true,
     "series_t": "Percy Jackson and the Olympians",
@@ -15,11 +17,13 @@
     ]
   },
   {
-    "id": "978-1423103349",
+    "id": 101,
+    "isbn": "978-1423103349",
     "name": "The Sea of Monsters",
     "author": "Rick Riordan",
-    "price": 6.49,
     "pages": 304,
+    "publishedYear": 2002,
+    "price": 6.49,
     "genres": "fantasy",
     "inStock": true,
     "series_t": "Percy Jackson and the Olympians",
@@ -30,11 +34,13 @@
     ]
   },
   {
-    "id": "978-1857995879",
+    "id": 102,
+    "isbn": "978-1857995879",
     "name": "Sophie's World : The Greek Philosophers",
     "author": "Jostein Gaarder",
-    "price": 3.07,
     "pages": 64,
+    "publishedYear": 2002,
+    "price": 3.07,
     "genres": "fantasy",
     "inStock": true,
     "sequence_i": 1,
@@ -44,11 +50,13 @@
     ]
   },
   {
-    "id": "978-1933988177",
+    "id": 103,
+    "isbn": "978-1933988177",
     "name": "Lucene in Action, Second Edition",
     "author": "Michael McCandless",
-    "price": 30.50,
     "pages": 475,
+    "publishedYear": 2002,
+    "price": 30.50,
     "genres": "IT",
     "inStock": true,
     "sequence_i": 1,

backend/src/main/resources/db/migration/U1__books.sql

@@ -0,0 +1 @@
+drop table books;

backend/src/main/resources/db/migration/V1__books.sql

@@ -0,0 +1,40 @@
+create table if not exists books (
+  id serial primary key,
+  isbn character varying(16) unique not null,
+  name text not null,
+  author text not null,
+  pages integer not null,
+  published_year integer not null,
+  created_at timestamp DEFAULT (now() at time zone 'utc') not null,
+  updated_at timestamp DEFAULT (now() at time zone 'utc') not null
+);
+
+INSERT INTO books(isbn, name, author, pages, published_year) VALUES
+  (
+    '978-0641723445',
+    'The Lightning Thief',
+    'Rick Riordan',
+    384,
+    2002
+  ),
+  (
+    '978-1423103349',
+    'The Sea of Monsters',
+    'Rick Riordan',
+    304,
+    1978
+  ),
+  (
+    '978-1857995879',
+    'Sophie''s World : The Greek Philosophers',
+    'Jostein Gaarder',
+      64,
+      2012
+),
+(
+  '978-1933988177',
+  'Lucene in Action, Second Edition',
+  'Michael McCandless',
+  475,
+  1564
+);

backend/src/main/scala/snorri/models/Book.scala

@@ -1,19 +1,25 @@
 package snorri.models
 
 import io.circe.generic.semiauto.*
-import io.circe.{Decoder, Encoder}
+import io.circe.{Decoder as CirceDecoder, Encoder as CirceEncoder}
+import skunk.*
+import skunk.codec.all.*
+import skunk.Decoder
 
-// NOTE: Not yet reading series_t,sequence_i and cat
 case class Book(
-  id:      String,
-  name:    String,
-  author:  String,
-  genres:  String,
-  inStock: Boolean,
-  price:   Double,
-  pages:   Int
+  id:            Int,
+  isbn:          String,
+  name:          String,
+  author:        String,
+  pages:         Int,
+  publishedYear: Int
 )
 
 object Book:
-  implicit val bookEncoder: Encoder[Book] = deriveEncoder[Book]
-  implicit val bookDecoder: Decoder[Book] = deriveDecoder[Book]
+  implicit val circeEncoder: CirceEncoder[Book] = deriveEncoder[Book]
+  implicit val circeDecoder: CirceDecoder[Book] = deriveDecoder[Book]
+
+  val skunkDecoder: Decoder[Book] =
+    (int4 ~ varchar(16) ~ text ~ text ~ int4 ~ int4).map { case id ~ isbn ~ name ~ author ~ pages ~ publishedYear =>
+      Book(id, isbn, name, author, pages, publishedYear)
+    }

backend/src/main/scala/snorri/repositories/BooksDb.scala

@@ -1,7 +1,12 @@
 package snorri.repositories
 
+import cats.effect.IO
+import cats.effect.unsafe.implicits.global
 import io.circe.parser.decode
-import krop.all.*
+import natchez.Trace.Implicits.noop
+import skunk.codec.all.varchar
+import skunk.implicits.sql
+import skunk.{Query, Session}
 import snorri.models.Book
 import snorri.utils.use
 
@@ -10,12 +15,43 @@ import scala.io.{BufferedSource, Source}
 object BooksDb:
   val all: Map[String, Book] = loadBooks()
 
-  def find(id: String): Option[Book] =
-    all.get(id)
+  def find(isbn: String): Option[Book] =
+    val query: Query[String, Book] =
+      sql"""SELECT id, isbn, name, author, pages, published_year
+            FROM   snorri.books
+            WHERE  isbn = $varchar""".query(Book.skunkDecoder)
+
+    // TODO: Using Session just to get it up and running.
+    //  Will have to use something like connection pool or transaction manager
+    Session
+      .single[IO](
+        host = "localhost",
+        port = 5432,
+        database = "snorri",
+        user = "postgres",
+        password = Some("password")
+      )
+      .use { s =>
+        for {
+          q <- s.prepare(query)
+          b <- q.unique(isbn)
+        } yield b
+      }
+      .attempt
+      .map {
+        case Right(book) =>
+          println(book)
+          Some(book)
+        case Left(err) =>
+          err.fillInStackTrace().printStackTrace()
+          None
+      }
+      .unsafeRunSync()
 
   private def loadBooks(): Map[String, Book] =
     decode[List[Book]](retrieveBooks()) match {
-      case Right(books) => books.map(b => b.id -> b).toMap
+      case Right(books) =>
+        books.map(b => b.isbn -> b).toMap
       case Left(err) =>
         err.fillInStackTrace().printStackTrace()
         throw err.getCause

build.sbt

@@ -1,3 +1,7 @@
+import scala.sys.process.*
+import org.flywaydb.core.Flyway
+import DbTasks.*
+
 Global / onChangedBuildSource := ReloadOnSourceChanges
 
 ThisBuild / scalaVersion    := "3.3.1"
@@ -7,6 +11,13 @@ ThisBuild / tlCiHeaderCheck := false
 // Remove dependency submission, which is failing and is not worth setting up for this project
 ThisBuild / githubWorkflowAddedJobs ~= (jobs => jobs.filter(job => job.id != "dependency-submission"))
 
+val startDb = taskKey[Unit]("Start the Postgres Docker container")
+val initDb  = taskKey[Unit]("Create database tables and add initial data")
+
+val databaseDirectory     = settingKey[File]("The directory on the local file system where the database is stored.")
+val databaseContainerName = settingKey[String]("The name of the Docker container running the database.")
+val databaseName          = settingKey[String]("The name of the Snorri database.")
+
 val commonSettings = Seq(
   libraryDependencies ++=
     "org.creativescala" %% "krop-core" % "0.7.0" ::
@@ -18,15 +29,43 @@ lazy val snorriRoot =
     .in(file("."))
     .aggregate(backend, frontend, integration)
 
-lazy val backend =
+lazy val backend: Project =
   project
     .in(file("backend"))
     .settings(
       commonSettings,
+      libraryDependencies += "org.tpolecat" %% "skunk-core" % "0.6.3",
       // This sets Krop into development mode, which gives useful tools for
       // developers. If you don't set this, Krop runs in production mode.
       run / javaOptions += "-Dkrop.mode=development",
-      run / fork := true
+      run / fork            := true,
+      databaseDirectory     := baseDirectory.value / "data",
+      databaseContainerName := "snorri-db",
+      databaseName          := "snorri",
+      startDb := {
+        s"mkdir ${databaseDirectory.value}".!
+        s"docker run --name ${databaseContainerName.value} -p 5432:5432 -v ${databaseDirectory.value}/snorri-db -e POSTGRES_PASSWORD=password -d postgres:latest".!
+      },
+      initDb := {
+        createDb(databaseName.value, databaseContainerName.value)
+        createTrgm(databaseName.value, databaseContainerName.value)
+
+        Flyway
+          .configure(getClass.getClassLoader)
+          .driver("org.postgresql.Driver")
+          .dataSource(
+            s"jdbc:postgresql://localhost:5432/${databaseName.value}",
+            "postgres",
+            "password"
+          )
+          .schemas(databaseName.value)
+          .locations(s"filesystem:${(Compile / resourceDirectory).value / "db" / "migration"}")
+          .validateMigrationNaming(true)
+          .failOnMissingLocations(true)
+          .outOfOrder(true)
+          .load
+          .migrate
+      }
     )
 
 lazy val frontend =
@@ -46,7 +85,7 @@ lazy val integration = (project in file("integration"))
     libraryDependencies ++=
       "com.dimafeng"    %% "testcontainers-scala-scalatest"  % "0.41.2" % Test ::
         "com.dimafeng"  %% "testcontainers-scala-postgresql" % "0.41.2" % Test ::
-        "org.postgresql" % "postgresql"                      % "42.5.1" % Test ::
+        "org.postgresql" % "postgresql"                      % "42.7.1" % Test ::
         "org.scalatest" %% "scalatest"                       % "3.2.18" % Test ::
         Nil
   )

project/DbTasks.scala

@@ -0,0 +1,37 @@
+import scala.sys.process.*
+
+object DbTasks {
+  def createDb(dbName: String, containerName: String): Unit = {
+    val dockerCmd = List(
+      "docker",
+      "exec",
+      containerName,
+      "su",
+      "-",
+      "postgres",
+      "-c",
+      s"createdb $dbName"
+    )
+
+    println(dockerCmd.mkString(" "))
+
+    (dockerCmd #|| List("echo", "Ignoring error and continuing")).!
+  }
+
+  def createTrgm(dbName: String, containerName: String): Unit = {
+    val cmd = List(
+      "docker",
+      "exec",
+      containerName,
+      "su",
+      "-",
+      "postgres",
+      "-c",
+      s"psql $dbName -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'"
+    )
+
+    println(cmd.mkString(" "))
+
+    (cmd #|| List("echo", "Ignoring error and continuing")).!
+  }
+}

project/Modules.scala

@@ -1,10 +1,15 @@
 import sbt.*
 
 object Modules {
-  val CirceVersion = "0.14.6"
+  val CirceVersion  = "0.14.6"
+  val FlywayVersion = "9.15.1"
 
   lazy val circe: List[ModuleID] =
     "io.circe"   %% "circe-parser"  % CirceVersion ::
       "io.circe" %% "circe-generic" % CirceVersion ::
       Nil
+
+  lazy val flyway: List[ModuleID] =
+    "org.flywaydb" % "flyway-core" % FlywayVersion :: Nil
+
 }

project/build.sbt

@@ -0,0 +1,5 @@
+libraryDependencies ++=
+  "org.flywaydb"     % "flyway-core"                % "10.9.0" ::
+    "org.postgresql" % "postgresql"                 % "42.7.2" ::
+    "org.flywaydb"   % "flyway-database-postgresql" % "10.8.1" ::
+    Nil