Docker/Test improvements (#5946)

.dockerignore

@@ -0,0 +1,4 @@
+.env
+icon_cache
+logs
+dist

.env.test

@@ -1,6 +1,5 @@
 TZ="UTC"
-ROBOCHIMP_DATABASE_URL="postgresql://postgres:postgres@localhost:5435/robochimp_integration_test?connection_limit=500&pool_timeout=0&schema=public"
-DATABASE_URL="postgresql://postgres:postgres@localhost:5435/osb_integration_test?connection_limit=500&pool_timeout=0&schema=public"
 CLIENT_ID=111398433321891634
 BOT_TOKEN=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-TEST=true
+TEST=true
+CI=true

.github/workflows/integration_tests.yml

@@ -9,37 +9,15 @@ on:
 
 jobs:
   test:
-    name: Node v${{ matrix.node_version }} - ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     timeout-minutes: 10
-    strategy:
-      matrix:
-        node_version: [20.15.0]
-        os: [ubuntu-latest]
 
     steps:
-      - name: Install System Packages
-        run: sudo apt-get install -y build-essential libpq-dev
-
       - name: Checkout Project
         uses: actions/checkout@v4
-      - run: corepack enable && corepack install
-      - name: Use Node.js ${{ matrix.node_version }}
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ matrix.node_version }}
-          cache: yarn
-      - name: Restore CI Cache
-        uses: actions/cache@v4
-        with:
-          path: node_modules
-          key: ${{ runner.os }}-${{ matrix.node_version }}-${{ hashFiles('**/yarn.lock') }}
-      - name: Install Dependencies
-        run: yarn --immutable
-      - name: Copy Configuration
-        run: |
-          pushd src &&
-          cp config.example.ts config.ts &&
-          popd && cp .env.test .env
-      - name: Test
-        run: yarn test:integration
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run Integration Tests
+        run: docker-compose up --build --abort-on-container-exit

docker-compose.yml

@@ -1,12 +1,27 @@
 services:
-  osb:
+  db:
     image: postgres:16-alpine
     command: -c 'max_connections=1000'
     restart: always
     container_name: osb_database
     ports:
-      - "5435:3001"
+      - "5435:5435"
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
-      PGPORT: 3001
+      PGPORT: 5435
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+  app:
+    build:
+      context: .
+    depends_on:
+      - db
+    environment:
+      ROBOCHIMP_DATABASE_URL: postgresql://postgres:postgres@db:5435/robochimp_integration_test?connection_limit=500&pool_timeout=0&schema=public
+      DATABASE_URL: postgresql://postgres:postgres@db:5435/osb_integration_test?connection_limit=500&pool_timeout=0&schema=public
+      WAIT_HOSTS: db:5435
+
+volumes:
+  postgres_data:

dockerfile

@@ -0,0 +1,35 @@
+FROM node:20.15.0-alpine AS base
+WORKDIR /usr/src/app
+ENV CI=true
+RUN apk add --no-cache dumb-init python3 g++ make git
+RUN corepack enable
+
+COPY yarn.lock package.json .yarnrc.yml ./
+
+ENTRYPOINT ["dumb-init", "--"]
+
+FROM base AS dependencies
+WORKDIR /usr/src/app
+RUN yarn remove zlib-sync && yarn install
+
+FROM base AS build-run
+WORKDIR /usr/src/app
+ENV NODE_ENV="development"
+ENV NODE_OPTIONS="--enable-source-maps --max_old_space_size=4096"
+
+COPY --from=dependencies /usr/src/app/node_modules /usr/src/app/node_modules
+COPY . .
+
+ENV CI=true
+RUN cp .env.test .env
+RUN cp src/config.example.ts src/config.ts
+
+ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.9.0/wait /wait
+RUN chmod +x /wait
+
+CMD /wait && \
+    yarn prisma db push --schema='./prisma/robochimp.prisma' && \
+    yarn prisma db push --schema='./prisma/schema.prisma' && \
+    yarn run build:esbuild && \
+    yarn vitest run --config vitest.integration.config.mts && \
+    exit 0

package.json

@@ -1,5 +1,6 @@
 {
 	"scripts": {
+		"watch": "nodemon -e ts -w src --exec 'yarn buildandrun'",
 		"build": "tsx ./src/scripts/build.ts",
 		"fix": "tsx ./src/scripts/troubleshooter.ts",
 		"start": "yarn build && node --enable-source-maps dist/",
@@ -9,12 +10,15 @@
 		"build:tsc": "tsc -p src",
 		"wipedist": "node -e \"try { require('fs').rmSync('dist', { recursive: true }) } catch(_){}\"",
 		"dev": "yarn && yarn wipedist && yarn lint && yarn build && yarn test && npm i -g dpdm && dpdm --exit-code circular:1 --progress=false --warning=false --tree=false ./dist/index.js",
-		"test": "concurrently --timings --raw --kill-others-on-fail \"tsc -p src\" \"yarn test:lint\" \"yarn test:unit\" \"tsc -p tests/integration\" \"tsc -p tests/unit\"",
+		"test": "concurrently --raw --kill-others-on-fail \"tsc -p src\" \"yarn test:lint\" \"yarn test:unit\" \"tsc -p tests/integration\" \"tsc -p tests/unit\"",
 		"test:lint": "biome check --diagnostic-level=error",
 		"test:unit": "vitest run --coverage --config vitest.unit.config.mts",
 		"test:watch": "vitest --config vitest.unit.config.mts --coverage",
 		"test:integration": "tsx ./src/scripts/integration-tests.ts",
-		"watch": "tsc -w -p src"
+		"buildandrun": "yarn build:esbuild && node dist",
+		"build:esbuild": "concurrently --raw \"yarn build:main\" \"yarn build:workers\"",
+		"build:main": "esbuild src/index.ts src/lib/workers/index.ts --minify --legal-comments=none --outdir=./dist --log-level=error --bundle --platform=node --loader:.node=file --external:@napi-rs/canvas --external:@prisma/robochimp --external:@prisma/client --external:zlib-sync --external:bufferutil --external:oldschooljs --external:discord.js --external:node-fetch --external:piscina",
+		"build:workers": "esbuild src/lib/workers/kill.worker.ts src/lib/workers/finish.worker.ts src/lib/workers/casket.worker.ts --log-level=error --bundle --minify --legal-comments=none --outdir=./dist/lib/workers --platform=node --loader:.node=file --external:@napi-rs/canvas --external:@prisma/robochimp --external:@prisma/client --external:zlib-sync --external:bufferutil --external:oldschooljs --external:discord.js --external:node-fetch --external:piscina"
 	},
 	"dependencies": {
 		"@napi-rs/canvas": "^0.1.53",
@@ -51,7 +55,9 @@
 		"@types/node-fetch": "^2.6.1",
 		"@vitest/coverage-v8": "^1.6.0",
 		"concurrently": "^8.2.2",
+		"esbuild": "0.21.5",
 		"fast-glob": "^3.3.2",
+		"nodemon": "^3.1.4",
 		"prettier": "^3.3.2",
 		"prisma": "^5.16.1",
 		"tsx": "^4.16.2",
@@ -61,5 +67,8 @@
 	"engines": {
 		"node": "20.15.0"
 	},
-	"packageManager": "[email protected]"
+	"packageManager": "[email protected]",
+	"resolutions": {
+		"esbuild": "0.21.5"
+	}
 }

src/index.ts

@@ -1,12 +1,9 @@
+import './lib/safeglobals';
 import './lib/globals';
-import './lib/data/itemAliases';
-import './lib/data/trophies';
 import './lib/crons';
 import './lib/MUser';
 import './lib/util/transactItemsFromBank';
-import './lib/itemMods';
 import './lib/geImage';
-import './lib/util/logger';
 
 import { MahojiClient } from '@oldschoolgg/toolkit';
 import { init } from '@sentry/node';

src/lib/bankImage.ts

@@ -11,8 +11,8 @@ import fetch from 'node-fetch';
 import { Bank } from 'oldschooljs';
 import type { Item } from 'oldschooljs/dist/meta/types';
 import { toKMB } from 'oldschooljs/dist/util/util';
-
 import { resolveItems } from 'oldschooljs/dist/util/util';
+
 import { BOT_TYPE, BitField, ItemIconPacks, PerkTier, toaPurpleItems } from '../lib/constants';
 import { allCLItems } from '../lib/data/Collections';
 import { filterableTypes } from '../lib/data/filterables';

src/lib/constants.ts

@@ -9,8 +9,6 @@ import * as dotenv from 'dotenv';
 import { getItemOrThrow, resolveItems } from 'oldschooljs/dist/util/util';
 import { z } from 'zod';
 
-import './data/itemAliases';
-
 import { DISCORD_SETTINGS, production } from '../config';
 import type { AbstractCommand } from '../mahoji/lib/inhibitors';
 import { SkillsEnum } from './skilling/types';
@@ -534,11 +532,12 @@ const globalConfigSchema = z.object({
 	clientID: z.string().min(10).max(25),
 	geAdminChannelID: z.string().default(''),
 	redisPort: z.coerce.number().int().optional(),
-	botToken: z.string().min(1)
+	botToken: z.string().min(1),
+	isCI: z.coerce.boolean().default(false)
 });
 dotenv.config({ path: path.resolve(process.cwd(), process.env.TEST ? '.env.test' : '.env') });
 
-if (!process.env.BOT_TOKEN) {
+if (!process.env.BOT_TOKEN && !process.env.CI) {
 	throw new Error(
 		`You need to specify the BOT_TOKEN environment variable, copy your bot token from your config.ts and put it in the ".env" file like so:\n\nBOT_TOKEN=your_token_here`
 	);
@@ -548,7 +547,8 @@ export const globalConfig = globalConfigSchema.parse({
 	clientID: process.env.CLIENT_ID,
 	geAdminChannelID: process.env.GE_ADMIN_CHANNEL_ID,
 	redisPort: process.env.REDIS_PORT,
-	botToken: process.env.BOT_TOKEN
+	botToken: process.env.BOT_TOKEN,
+	isCI: process.env.CI
 });
 
 export const ONE_TRILLION = 1_000_000_000_000;

src/lib/data/itemAliases.ts

@@ -1,6 +1,11 @@
+import { modifyItem } from '@oldschoolgg/toolkit';
 import { Items } from 'oldschooljs';
+import { allTeamCapes } from 'oldschooljs/dist/data/itemConstants';
 import { itemNameMap } from 'oldschooljs/dist/structures/Items';
 import { cleanString } from 'oldschooljs/dist/util/cleanString';
+import { resolveItems } from 'oldschooljs/dist/util/util';
+
+import { getOSItem } from '../util/getOSItem';
 
 export function setItemAlias(id: number, name: string | string[], rename = true) {
 	const existingItem = Items.get(id);
@@ -298,3 +303,90 @@ setItemAlias(25_922, 'Antique lamp (hard ca)');
 setItemAlias(25_923, 'Antique lamp (elite ca)');
 setItemAlias(25_924, 'Antique lamp (master ca)');
 setItemAlias(25_925, 'Antique lamp (grandmaster ca)');
+
+/**
+ * Trophies
+ */
+
+// BSO (Twisted) trophies
+setItemAlias(24_372, 'BSO dragon trophy');
+setItemAlias(24_374, 'BSO rune trophy');
+setItemAlias(24_376, 'BSO adamant trophy');
+setItemAlias(24_378, 'BSO mithril trophy');
+setItemAlias(24_380, 'BSO steel trophy');
+setItemAlias(24_382, 'BSO iron trophy');
+setItemAlias(24_384, 'BSO bronze trophy');
+
+// Comp. trophies
+setItemAlias(25_042, 'Comp. dragon trophy');
+setItemAlias(25_044, 'Comp. rune trophy');
+setItemAlias(25_046, 'Comp. adamant trophy');
+setItemAlias(25_048, 'Comp. mithril trophy');
+setItemAlias(25_050, 'Comp. steel trophy');
+setItemAlias(25_052, 'Comp. iron trophy');
+setItemAlias(25_054, 'Comp. bronze trophy');
+
+// Placeholder trophies
+setItemAlias(26_515, 'Placeholder dragon trophy');
+setItemAlias(26_513, 'Placeholder rune trophy');
+setItemAlias(26_511, 'Placeholder adamant trophy');
+setItemAlias(26_509, 'Placeholder mithril trophy');
+setItemAlias(26_507, 'Placeholder steel trophy');
+setItemAlias(26_505, 'Placeholder iron trophy');
+setItemAlias(26_503, 'Placeholder bronze trophy');
+
+export const allTrophyItems = resolveItems([
+	'BSO dragon trophy',
+	'BSO rune trophy',
+	'BSO adamant trophy',
+	'BSO mithril trophy',
+	'BSO steel trophy',
+	'BSO iron trophy',
+	'BSO bronze trophy',
+	'Comp. dragon trophy',
+	'Comp. rune trophy',
+	'Comp. adamant trophy',
+	'Comp. mithril trophy',
+	'Comp. steel trophy',
+	'Comp. iron trophy',
+	'Comp. bronze trophy',
+	'Placeholder dragon trophy',
+	'Placeholder rune trophy',
+	'Placeholder adamant trophy',
+	'Placeholder mithril trophy',
+	'Placeholder steel trophy',
+	'Placeholder iron trophy',
+	'Placeholder bronze trophy'
+]);
+
+for (const item of allTrophyItems) {
+	modifyItem(item, {
+		tradeable: false,
+		tradeable_on_ge: false,
+		customItemData: {
+			cantBeSacrificed: true
+		}
+	});
+}
+
+/**
+ * Item modifications
+ */
+
+export interface CustomItemData {
+	cantBeSacrificed?: true;
+}
+declare module 'oldschooljs/dist/meta/types' {
+	interface Item {
+		customItemData?: CustomItemData;
+	}
+}
+
+for (const item of allTeamCapes) {
+	modifyItem(item.id, {
+		price: 100
+	});
+	if (getOSItem(item.id).price !== 100) {
+		throw new Error(`Failed to modify price of item ${item.id}`);
+	}
+}