fix: docker compose failing to build and spark failing to start

- moved isTestMode from the tests.ts library to the server.ts library to circumvent any import errors.
- programmatically import in-memory database only if the server has been detected to be running in test mode.
- fix creation of citext extension on the in-memory database by including a statement before migration.
- programmatically import the redis memory server only if the server is running in the test mode.
- added build-essential to the Dockerfile to allow for the make command to work when installing dependencies.
- made the spark docker container depend on the database and keydb containers.
- created a postgres spark user for spark related tasks instead of using the root/postgres user.

Dockerfile

@@ -6,6 +6,8 @@ WORKDIR /usr/src/app
 FROM base AS install
 RUN mkdir -p /temp/dev
 COPY package.json bun.lockb /temp/dev/
+# required in order for redis-memory-store to be installed
+RUN apt-get update && apt-get install -y build-essential
 RUN cd /temp/dev && bun install --frozen-lockfile
 
 # install with --production (exclude devDependencies)

docker-compose.yml

@@ -8,6 +8,9 @@ services:
             - '3005:3005'
         volumes:
             - ./.env:/usr/src/app/.env
+        depends_on:
+            - postgres
+            - keydb
         restart: unless-stopped
         networks:
             - spark
@@ -26,6 +29,7 @@ services:
         container_name: postgres
         environment:
             - POSTGRES_DB=spark
+            - POSTGRES_USER=spark
             - POSTGRES_PASSWORD=postgres
         volumes:
             - ./docker-data/postgres_data:/var/lib/postgresql/data

src/drizzle/db.ts

@@ -1,14 +1,31 @@
-import { PGlite as testClient } from '@electric-sql/pglite'; // for unit tests use
 import { drizzle as prodDrizzle } from 'drizzle-orm/node-postgres'; // for production use
-import { drizzle as testDrizzle } from 'drizzle-orm/pglite'; // for unit tests use
 import { Client as prodClient } from 'pg'; // for production use
 import * as schema from './schema'; // get all the schema
 
 export const prodDbClient = new prodClient({
 	connectionString: process.env.DATABASE_URL as string
 });
 
-export const db =
-	process.env.NODE_ENV !== 'test'
-		? prodDrizzle(prodDbClient, { schema }) // we want to use a real database
-		: testDrizzle(new testClient(), { schema }); // we want to use a in-memory database
+import { NodePgDatabase } from 'drizzle-orm/node-postgres';
+import { PgliteDatabase } from 'drizzle-orm/pglite';
+
+let db:
+	| (NodePgDatabase<typeof schema> & { $client: prodClient })
+	| (PgliteDatabase<typeof schema> & { $client: any });
+
+if (process.env.NODE_ENV !== 'test') {
+	db = prodDrizzle(prodDbClient, { schema }); // we want to use a real database
+} else {
+	import('@electric-sql/pglite').then(({ PGlite: testClient }) => {
+		import('drizzle-orm/pglite').then(({ drizzle: testDrizzle }) => {
+			// @ts-ignore - TODO: fix later
+			import('@electric-sql/pglite/contrib/citext').then(({ citext }) => {
+				db = testDrizzle(new testClient({ extensions: { citext } }), {
+					schema
+				}); // we want to use an in-memory database
+			});
+		});
+	});
+}
+
+export { db };

src/lib/server.ts

@@ -11,7 +11,6 @@ import { postMedia, user } from '../drizzle/schema';
 import { syncClient, tokenClient } from '../redis';
 import { authorize } from './authentication';
 import { convertModelToUser, getHashedPk, internalUsers } from './authentik';
-import { isTestMode } from './tests';
 
 /**
  * "Legacy" endpoint for uploading media.
@@ -368,4 +367,12 @@ async function createUsersFromRedisTokens() {
 	);
 }
 
-export { createUsersFromRedisTokens, mediaUploadEndpoint, webhookEndpoint };
+// Just a shortcut for checking if we are in test mode.
+const isTestMode = process.env.NODE_ENV === 'test';
+
+export {
+	createUsersFromRedisTokens,
+	mediaUploadEndpoint,
+	webhookEndpoint,
+	isTestMode
+};

src/lib/tests.ts

@@ -65,6 +65,3 @@ export async function removeUser(sub: string) {
 
 	return tokenClient.del(`tokens:${sub}`);
 }
-
-// Just a shortcut for checking if we are in test mode.
-export const isTestMode = process.env.NODE_ENV === 'test';

src/redis.ts

@@ -1,10 +1,13 @@
 import { createClient } from 'redis';
-import RedisMemoryServer from 'redis-memory-server';
-import { isTestMode } from './lib/tests';
+import { isTestMode } from './lib/server';
 
 if (isTestMode) {
-	const testRedisClient = new RedisMemoryServer({ instance: { port: 1234 } });
-	testRedisClient.start();
+	import('redis-memory-server').then(({ default: RedisMemoryServer }) => {
+		const testRedisClient = new RedisMemoryServer({
+			instance: { port: 1234 }
+		});
+		testRedisClient.start();
+	});
 }
 
 // Create a Redis client using the createClient function.

src/server.ts

@@ -12,7 +12,7 @@ import {
 	mediaUploadEndpoint,
 	webhookEndpoint
 } from './lib/server';
-import { isTestMode } from './lib/tests';
+import { isTestMode } from './lib/server';
 import { redisClient, syncClient, tokenClient } from './redis';
 import { schema } from './schema';
 import { enableAll } from './lib/logger';
@@ -94,6 +94,7 @@ export async function startServer() {
 		await createUsersFromRedisTokens();
 	} else {
 		// Migrate the database.
+		await db.execute(sql`CREATE EXTENSION IF NOT EXISTS citext;`);
 		await migrate(db, { migrationsFolder: './drizzle' });
 	}