fix(s3): remove s3 use in service, update litestream and add to docker-compose

.env.test

@@ -6,4 +6,3 @@ LOG_FORMAT=simple
 NODE_VERSION=18.17.0
 BLOCKLIST_CONTRACT_IDS=''
 PREFETCH_CONTRACTS=false
-BOOTSTRAP_CONTRACTS=false

README.md

@@ -36,6 +36,19 @@ docker run -e GATEWAY_HOST=arweave.net -p 3000:3000 ghcr.io/ar-io/arns-service:l
 
 You can run on a different port by changing the `-e PORT=3000 -p 3000:3000` to `-e PORT=4000 -p 4000:4000`, for example, or specify a `.env` file with `--env-file` flag.
 
+#### Profiles
+
+The service can be run with different profiles to facilitate different environments. The following profiles are supported:
+
+- `test`: runs the service and other containers for executing integration tests
+- `litestream`: runs the service with [Litestream] for syncing SQLite data to S3
+
+To run the service with a profile, you can use the `--profile` flag:
+
+```shell
+docker compose --env-file .env --profile litestream up --build
+```
+
 ## Warp
 
 The service leverages [Warp SDK] to retrieve, evaluate and cache contract state. To request a contract state, run:
@@ -92,7 +105,6 @@ The service can be configured using environment variables. The following environ
 - `PREFETCH_CONTRACTS`: boolean to enable/disable prefetching of contracts on startup. Defaults to `true`.
 - `PREFETCH_CONTRACT_IDS`: comma separated list of contract IDs to prefetch on startup
 - `ARNS_CONTRACT_TX_ID`: the ArNS contract transaction ID. Defaults to `bLAgYxAdX2Ry-nt6aH2ixgvJXbpsEYm28NgJgyqfs-U` and when `PREFETCH_CONTRACTS` is `true`, will be prefetched on startup.
-- `BOOTSTRAP_CACHE`: loads warp cache from S3 on startup. Defaults to `false`.
 - `BLOCKLISTED_CONTRACT_IDS`: comma separated list of contract IDs to block evaluation. These contracts will return `403` when requested.
 
 You can `cp .env.sample .env` and modify them locally.
@@ -127,11 +139,7 @@ yarn test:docker
 http://localhost:3000/api-docs
 ```
 
-For production, the Swagger UI is available at:
-
-```shell
-https://api.arns.app/api-docs
-```
+For production, the Swagger UI is available at [https://api.arns.app/api-docs](https://api.arns.app/api-docs)
 
 ## Contributions
 
@@ -150,3 +158,4 @@ https://api.arns.app/api-docs
 [Warp SDK]: https://github.com/warp-contracts/warp
 [warp-contracts-sqlite]: https://github.com/warp-contracts/warp-contracts-sqlite
 [D.R.E nodes]: https://academy.warp.cc/docs/dre/overview
+[Litestream]: https://litestream.io/

docker-compose.yaml

@@ -1,7 +1,6 @@
-version: '3'
-
 services:
   arlocal:
+    profiles: ['test']
     image: textury/arlocal:v1.1.35
     ports:
       - '1984:1984'
@@ -22,8 +21,26 @@ services:
       WAIT_TIME_SECONDS: ${WAIT_TIME_SECONDS:-}
     ports:
       - '3000:3000'
+    volumes:
+      - ${WARP_SQLITE_CACHE_PATH:-./cache/warp/sqlite}:/usr/src/app/cache/sqlite
+
+  litestream:
+    profiles: ['litestream']
+    build:
+      context: litestream
+    environment:
+      TVAL_LOCAL_DB_PATH: ${LITESTREAM_LOCAL_DB_PATH:-/app/data/sqlite}
+      TVAL_S3_BUCKET_PATH: ${LITESTREAM_S3_BUCKET_PATH:-}
+      TVAL_S3_BUCKET: ${LITESTREAM_S3_BUCKET:-}
+      TVAL_AWS_ACCESS_KEY_ID: ${LITESTREAM_AWS_ACCESS_KEY_ID:-}
+      TVAL_AWS_SECRET_ACCESS_KEY: ${LITESTREAM_AWS_SECRET_ACCESS_KEY:-}
+      WAIT_TIME_SECONDS: ${LITESTREAM_WAIT_TIME_SECONDS:-}
+      RESTORE_FROM_BACKUP: ${LITESTREAM_RESTORE_FROM_BACKUP:-false}
+    volumes:
+      - ${WARP_SQLITE_CACHE_PATH:-./cache/warp/sqlite}:/app/data/sqlite
 
   test-runner:
+    profiles: ['test']
     build:
       context: .
       dockerfile: tests/integration/Dockerfile.integration

litestream/.env.sample

@@ -0,0 +1,7 @@
+TVAL_LOCAL_DB_PATH=/cache/warp/sqlite
+TVAL_AWS_ACCESS_KEY_ID=
+TVAL_AWS_SECRET_ACCESS_KEY=
+TVAL_S3_BUCKET_PATH=/<sqlite-path-in-s3/
+TVAL_S3_BUCKET=
+WAIT_TIME_SECONDS=0
+RESTORE_FROM_BACKUP=false

litestream/Dockerfile

@@ -1,16 +1,9 @@
-# Start with a standard Ubuntu base image
-FROM ubuntu:20.04 as builder
+FROM litestream/litestream:0.3.13
 
-# Install curl and other necessary tools
-RUN apt-get update && apt-get install -y curl
+RUN apk update && apk add --no-cache curl
 
-# Download and install ytt
-RUN curl -L https://github.com/vmware-tanzu/carvel-ytt/releases/download/v0.44.1/ytt-linux-amd64 > /usr/local/bin/ytt \
-    && chmod 755 /usr/local/bin/ytt
-
-FROM litestream/litestream:latest
-
-COPY --from=builder /usr/local/bin/ytt /usr/local/bin/ytt
+COPY install-ytt.sh /install-ytt.sh
+RUN /install-ytt.sh && rm /install-ytt.sh
 
 COPY litestream.template.yaml .
 COPY docker-entrypoint.sh .

litestream/docker-entrypoint.sh

@@ -4,17 +4,21 @@ set -e
 # Update env vars
 ytt --data-values-env TVAL -f /litestream.template.yaml >  /litestream.yml
 
-# restore if needed
-echo "Restoring database from backup..."
-if ! /usr/local/bin/litestream restore -if-db-not-exists -if-replica-exists -parallelism 10 -config /litestream.yml $TVAL_LOCAL_DB_PATH/state.db; then
-    echo "Failed to restore state.db"
-fi
-if ! /usr/local/bin/litestream restore -if-db-not-exists -if-replica-exists -parallelism 10 -config /litestream.yml $TVAL_LOCAL_DB_PATH/contract.db; then
-    echo "Failed to restore contract.db"
+# only restore if environment variable is set
+if [[ "$RESTORE_FROM_BACKUP" = "true" ]]; then
+    echo "Restoring from backup..."
+    if ! /usr/local/bin/litestream restore -if-db-not-exists -if-replica-exists -parallelism 10 -config /litestream.yml $TVAL_LOCAL_DB_PATH/state.db; then
+     echo "Failed to restore state.db"
+    fi
+    if ! /usr/local/bin/litestream restore -if-db-not-exists -if-replica-exists -parallelism 10 -config /litestream.yml $TVAL_LOCAL_DB_PATH/contract.db; then
+        echo "Failed to restore contract.db"
+    fi
+else
+    echo "No RESTORE_FROM_BACKUP environment variable set, skipping restore..."
 fi
 
 if [[ -n "$WAIT_TIME_SECONDS" ]]; then
-    # Sleep for the number of seconds specified in WAIT_TIME_SECONDS
+    # Sleep for the number of seconds specified in WAIT_TIME_SECONDS, this is useful if we are prefetching contracts
     echo "Waiting for $WAIT_TIME_SECONDS seconds before starting replication..."
     sleep $WAIT_TIME_SECONDS
 fi

litestream/install-ytt.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+case $(uname -m) in
+    "aarch64")
+        echo "aarch64"
+        curl -L https://github.com/vmware-tanzu/carvel-ytt/releases/download/v0.49.0/ytt-linux-arm64 > /usr/local/bin/ytt
+        ;;
+    "x86_64")
+        echo "x86_64"
+        curl -L https://github.com/vmware-tanzu/carvel-ytt/releases/download/v0.49.0/ytt-linux-amd64 > /usr/local/bin/ytt
+        ;;
+    *)
+        echo "Unknown arch"
+        exit 1
+        ;;
+esac
+
+chmod 755 /usr/local/bin/ytt

litestream/litestream.template.yaml

@@ -1,9 +1,18 @@
 #@ load("@ytt:data", "data")
 
+#@ if data.values.AWS_ACCESS_KEY_ID != "":
+access-key-id: #@ data.values.AWS_ACCESS_KEY_ID
+secret-access-key: #@ data.values.AWS_SECRET_ACCESS_KEY
+#@ end
+
 dbs:
   - path: #@ data.values.LOCAL_DB_PATH + "/state.db"
     replicas:
-      - url: #@ data.values.S3_BUCKET_PATH
+      - type: s3
+        bucket: #@ data.values.S3_BUCKET
+        path: #@ data.values.S3_BUCKET_PATH
   - path: #@ data.values.LOCAL_DB_PATH + "/contract.db"
     replicas:
-      - url: #@ data.values.S3_BUCKET_PATH
+      - type: s3
+        bucket: #@ data.values.S3_BUCKET
+        path: #@ data.values.S3_BUCKET_PATH

package.json

@@ -17,11 +17,12 @@
     "start:prod": "yarn build && node dist/app.js",
     "build": "yarn clean && yarn && npx tsc --project ./tsconfig.json",
     "clean": "rimraf [ node_modules dist cache ]",
-    "test:docker": "docker compose --env-file=.env.test up --exit-code-from test-runner --build",
+    "test:docker": "docker compose --env-file=.env.test --profile test up --exit-code-from test-runner --build",
     "test:integration": "mocha --spec=tests/integration/**.test.ts",
     "test:integration:local": "docker compose up arlocal -d ; yarn test:integration $*; docker compose down -v",
     "docker:run": "docker compose up arns-service --build",
-    "docker:integration": "docker compose up --exit-code-from test-runner --build",
+    "docker:run:litestream": "docker compose --profile litestream up --build",
+    "docker:integration": "docker compose --profile test up --exit-code-from test-runner --build",
     "format:fix": "yarn prettier --write .",
     "format:check": "yarn prettier . --check",
     "lint:fix": "yarn eslint . --fix",
@@ -30,8 +31,6 @@
   },
   "dependencies": {
     "@ardrive/ardrive-promise-cache": "^1.1.4",
-    "@aws-sdk/client-s3": "^3.490.0",
-    "@aws-sdk/lib-storage": "^3.554.0",
     "@koa/cors": "^4.0.0",
     "@koa/router": "^12.0.0",
     "arweave": "^1.14.4",
@@ -42,7 +41,7 @@
     "lodash": "^4.17.21",
     "p-limit": "3.1.0",
     "prom-client": "^14.2.0",
-    "warp-contracts": "1.4.43",
+    "warp-contracts": "1.4.39",
     "warp-contracts-lmdb": "^1.1.10",
     "warp-contracts-sqlite": "^1.0.2",
     "winston": "^3.8.2",

src/api/warp.ts

@@ -86,7 +86,7 @@
 > = new ReadThroughPromiseCache({
   cacheParams: {
     cacheCapacity: 10_000,
-    cacheTTL: 1000 * 30, // 30 seconds
+    cacheTTL: 1000 * 60, // 60 seconds
   },
   readThroughFunction: readThroughToContractState,
 });
@@ -109,7 +109,7 @@
   constructor(
     public readonly contractTxId: string,
     public readonly functionName: string,
     public readonly input: any,
     public readonly warp: Warp,
     public readonly evaluationOptions: Partial<EvaluationOptions>,
     public readonly sortKey?: string,
@@ -141,7 +141,7 @@
 > = new ReadThroughPromiseCache({
   cacheParams: {
     cacheCapacity: 1_000,
-    cacheTTL: 1000 * 30, // 30 seconds
+    cacheTTL: 1000 * 60, // 60 seconds
   },
   readThroughFunction: readThroughToContractReadInteraction,
 });
@@ -314,8 +314,8 @@
     return new NotFoundError(error.message);
   } else if (
     (error instanceof Error &&
       (error as any).type &&
       ['TX_NOT_FOUND', 'TX_INVALID'].includes((error as any).type)) ||
     (typeof error === 'string' && (error as string).includes('TX_INVALID'))
   ) {
     return new NotFoundError(`Contract not found. ${error}`);
@@ -512,7 +512,7 @@
   input: ParsedUrlQuery;
   sortKey?: string | undefined;
 }): Promise<{
   result: any;
   evaluationOptions: Partial<EvaluationOptions>;
 }> {
   try {

src/constants.ts

@@ -17,8 +17,6 @@
 import { EvaluationOptions } from 'warp-contracts';
 
 export const PREFETCH_CONTRACTS = process.env.PREFETCH_CONTRACTS === 'true';
-export const BOOTSTRAP_CACHE = process.env.BOOTSTRAP_CACHE === 'true';
-export const SAVE_CACHE_TO_S3 = process.env.SAVE_CACHE_TO_S3 === 'true';
 export const BLOCKLISTED_CONTRACT_IDS = new Set(
   process.env.BLOCKLISTED_CONTRACT_IDS
     ? process.env.BLOCKLISTED_CONTRACT_IDS.split(',')

src/middleware/warp.ts

@@ -47,7 +47,7 @@ export const warp = WarpFactory.forMainnet(
         dbLocation: `./cache/warp/sqlite/state`,
       },
       {
-        maxEntriesPerContract: 1_000_000,
+        maxEntriesPerContract: 100_000,
       },
     ),
   )