AG-1498: improvements from validation

.github/workflows/ci.yml

@@ -215,7 +215,7 @@ jobs:
       - name: start database, server, and app
         run: |
           echo "==> start database and data containers"
-          docker-compose -f "${{ env.DOCKER_COMPOSE_PATH }}" up -d
+          docker compose -f "${{ env.DOCKER_COMPOSE_PATH }}" up -d
 
           echo "==> wait for data container to finish seeding database container"
           DATA_CONTAINER=$(docker compose -f "${{ env.DOCKER_COMPOSE_PATH }}" ps -a --format '{{.Name}}' mongo-seed)
@@ -237,7 +237,7 @@ jobs:
           pid=$(lsof -i :8080 -t) && kill ${pid}
 
           echo "==> stop database"
-          docker-compose -f "${{ env.DOCKER_COMPOSE_PATH }}" down
+          docker compose -f "${{ env.DOCKER_COMPOSE_PATH }}" down
 
       - uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}

README.md

@@ -11,12 +11,13 @@ What you need to run this app:
 - `node` and `npm` (`brew install node`)
 
   - Ensure you're running the latest versions Node `v16.x.x`+ and NPM `8.x.x`+
-- A version 6.0+ [MongoDB](https://www.mongodb.com/docs/manual/administration/install-community/) instance running on your local machine
-
-- You can optionally use a GUI like [Compass](https://www.mongodb.com/docs/compass/current/) or [Studio3T](https://studio3t.com/knowledge-base/articles/installation/) with your local database
-
+- If running against a local database: 
+  - A version 6.0+ [MongoDB](https://www.mongodb.com/docs/manual/administration/install-community/) instance running on your local machine
+  - You can optionally use a GUI like [Compass](https://www.mongodb.com/docs/compass/current/) or [Studio3T](https://studio3t.com/knowledge-base/articles/installation/) with your local database
   - Note that only Studio3T is compatible with the [AWS DocumentDB](https://docs.aws.amazon.com/documentdb/latest/developerguide/what-is.html) instances in Agora's dev, stage and prod environments. Either GUI tool will work with your local Mongo instance.
-
+- If running against a conatinerized database:
+  - [Docker Desktop](https://docs.docker.com/desktop/)
+  - [jq](https://jqlang.github.io/jq/) (`brew install jq`)
 
 > If you have `nvm` installed, which is highly recommended (`brew install nvm`) you can do a `nvm install --lts && nvm use` in `$` to run with the latest Node LTS. You can also have this `zsh` done for you [automatically](https://github.com/creationix/nvm#calling-nvm-use-automatically-in-a-directory-with-a-nvmrc-file)
 
@@ -170,9 +171,9 @@ Go to [http://localhost:8080](http://localhost:8080)
 
 ### 6 - Use containerized database
 
-1. Install Docker, if necessary.
+1. Install [Docker Desktop](https://docs.docker.com/desktop/) and [jq](https://jqlang.github.io/jq/) (`brew install jq`), if necessary.
 2. Update `data-file` and `data-version` in `package.json` to reflect the desired data release version, if necessary.
-3. Create an environment file: `npm run create-env`. 
+3. Create an environment file: `npm run create-env`. The credentials for connecting to the containerized MongoDB will be included in the generated `.env` file, where the connection string will be: `mongodb://{DB_USER}:{DB_PASS}@localhost:{DB_PORT}`.
 4. Start the containerized database: `npm run docker:db:start`. The necessary images will be pulled from GHCR. If you would like to use a different image, update DATA_IMAGE_PATH in .env. If the desired image does not exist, see steps below to create the desired image.
 5. Run the server and app against the containerized database: `npm run docker:dev`.
 6. Stop the containerized database: `npm run docker:db:stop`.
@@ -185,14 +186,15 @@ The `sage-bionetworks` package will contain images for data releases that have b
 
 If a dev needs to create an image for a data release that does not yet exist in the Sage-Bionetworks package, they should follow these steps: 
 
-1. Create a new branch.
-2. Update the `package.json` to reflect the appropriate `data-file` and `data-version` files.
-3. If necessary, update `./scripts/collections.csv` to specify new collections and `./scripts/mongo-create-Indexes.js` to specify new indexes.
-4. Commit the changes.
-5. Push the changes to your remote fork to trigger a run of the `ci.yml` workflow. 
-6. The new image will be available in your user namespaced GHCR package, e.g. `https://github.com/hallieswan/Agora/pkgs/container/agora-data-nonmonorepo`.
-7. Update your local `.env` file so `DATA_IMAGE_PATH` points to the newly created image, e.g. `ghcr.io/hallieswan/agora-data-nonmonorepo:syn13363290.68`.
-8. Start the containerized database: `npm run docker:db:start`.
+1. Set up GitHub secrets/variables in your remote fork as described [here](docker/README.md#workflow-setup), if necessary.
+2. Create a new branch.
+3. Update the `package.json` to reflect the appropriate `data-file` and `data-version` files.
+4. If necessary, update `./scripts/collections.csv` to specify new collections and `./scripts/mongo-create-Indexes.js` to specify new indexes.
+5. Commit the changes.
+6. Push the changes to your remote fork to trigger a run of the `ci.yml` workflow. 
+7. The new image will be available in your user namespaced GHCR package, e.g. `https://github.com/hallieswan/Agora/pkgs/container/agora-data-nonmonorepo`.
+8. Update your local `.env` file so `DATA_IMAGE_PATH` points to the newly created image, e.g. `ghcr.io/hallieswan/agora-data-nonmonorepo:syn13363290.68`.
+9. Start the containerized database: `npm run docker:db:start`.
 
 # Development
 

docker/README.md

@@ -7,15 +7,15 @@ Files in this directory:
 
 ## Workflow Setup
 
-The following secrets and variables need to be set up in GitHub for the `ci.yml` workflow to create the `agora-data-nonmonorepo` Docker image:
+The following secrets and variables need to be set up in GitHub for the `ci.yml` workflow to create the `agora-data-nonmonorepo` Docker image. See instructions for creating a GitHub environment and adding secrets/variables [here](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#creating-an-environment).
 
 e2e Environment secrets:
 
-| Variable           | Description                                | Example           |
-| ------------------ | ------------------------------------------ | ----------------- |
-| DB_USER            | The database user                          | dbuser            |
-| DB_PASS            | The database password                      | supersecret       |
-| SYNAPSE_AUTH_TOKEN | The Synapse service user view/download PAT | token-string-here |
+| Secret             | Description                                            | Example           |
+| ------------------ | ------------------------------------------------------ | ----------------- |
+| DB_USER            | The database user                                      | dbuser            |
+| DB_PASS            | The database password                                  | supersecret       |
+| SYNAPSE_AUTH_TOKEN | View/download PAT for `synapse-service-agora-e2e` user | token-string-here |
 
 e2e Environment variables:
 

package.json

@@ -41,11 +41,11 @@
     "e2e": "playwright test --trace on",
     "e2e:ui": "playwright test --ui",
     "e2e:update": "npm install -D @playwright/test@latest; npx playwright install --with-deps; npx playwright --version",
-    "create-env": "DB_PASS=$(openssl rand -base64 12) DATA_IMAGE_TAG=$(npm run docker:data-image-tag -s) envsubst < .env.example > .env",
+    "create-env": "[ ! -f .env ] && DB_PASS=$(openssl rand -base64 12) DATA_IMAGE_TAG=$(npm run docker:data-image-tag -s) envsubst < .env.example > .env",
     "docker:data-image-tag": "jq -r '.\"data-file\" + \".\" + .\"data-version\"' package.json",
     "docker:db:seed-complete": "echo 'Waiting for docker db to finish loading data (~30s)....'; DATA_CONTAINER=$(docker compose --env-file .env -f ./docker/docker-compose.yml ps -a --format '{{.Name}}' mongo-seed); docker wait ${DATA_CONTAINER}",
-    "docker:db:start": "docker-compose --env-file .env -f ./docker/docker-compose.yml up -d; npm run docker:db:seed-complete",
-    "docker:db:stop": "docker-compose --env-file .env -f ./docker/docker-compose.yml down",
+    "docker:db:start": "docker compose --env-file .env -f ./docker/docker-compose.yml up -d; npm run docker:db:seed-complete",
+    "docker:db:stop": "docker compose --env-file .env -f ./docker/docker-compose.yml down",
     "docker:dev": "set -a; source .env; set +a; MONGODB_HOST=\"localhost\" MONGODB_PORT=\"${DB_PORT}\" APP_ENV=\"e2e\" npm run dev"
   },
   "pre-commit": [