From c6aed06c036f37a1df8df6d0ad3b2c350c3894ed Mon Sep 17 00:00:00 2001 From: Adrian Monge Date: Fri, 21 Jun 2024 22:08:43 +0200 Subject: [PATCH] (BPD-1443) Implement GitHub action workflows integrated with Pulumi (#3) * task: Add .secrets to .gitignore * task: Add the two workflows and requirements file * task: Specify develop branch for preview workflow * task: Remove unnecessary workflow files * task: Secrets manager retrieval for preview workflow * task: Remove AWS CLI installation from GH Actions * task: Remove AWS CLI installation from deploy WF --- .github/workflows/_lint.yml | 72 ---------------- .github/workflows/build_deploy_image.yml | 37 -------- .github/workflows/ci.yml | 105 ----------------------- .github/workflows/deploy-dev.yml | 49 +++++++++++ .github/workflows/preview-dev.yml | 49 +++++++++++ .gitignore | 3 + requirements.txt | 5 ++ 7 files changed, 106 insertions(+), 214 deletions(-) delete mode 100644 .github/workflows/_lint.yml delete mode 100644 .github/workflows/build_deploy_image.yml delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/deploy-dev.yml create mode 100644 .github/workflows/preview-dev.yml create mode 100644 requirements.txt diff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.yml deleted file mode 100644 index 46b6e191..00000000 --- a/.github/workflows/_lint.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: lint - -on: - workflow_call: - inputs: - working-directory: - required: true - type: string - description: "From which folder this pipeline executes" - -env: - POETRY_VERSION: "1.5.1" - WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }} - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - # Only lint on the min and max supported Python versions. - # It's extremely unlikely that there's a lint issue on any version in between - # that doesn't show up on the min or max versions. - # - # GitHub rate-limits how many jobs can be running at any one time. - # Starting new jobs is also relatively slow, - # so linting on fewer versions makes CI faster. - python-version: - - "3.9" - - "3.11" - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} - uses: "./.github/actions/poetry_setup" - with: - python-version: ${{ matrix.python-version }} - poetry-version: ${{ env.POETRY_VERSION }} - working-directory: ${{ inputs.working-directory }} - cache-key: lint-with-extras - - - name: Check Poetry File - shell: bash - working-directory: ${{ inputs.working-directory }} - run: | - poetry check - - - name: Check lock file - shell: bash - working-directory: ${{ inputs.working-directory }} - run: | - poetry lock --check - - - name: Install dependencies - # Also installs dev/lint/test/typing dependencies, to ensure we have - # type hints for as many of our libraries as possible. - # This helps catch errors that require dependencies to be spotted, for example: - # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341 - # - # If you change this configuration, make sure to change the `cache-key` - # in the `poetry_setup` action above to stop using the old cache. - # It doesn't matter how you change it, any change will cause a cache-bust. - working-directory: ${{ inputs.working-directory }} - run: | - poetry install --with dev,lint,test - # Add typing dependencies once we roll out mypy - # poetry install --with dev,lint,test,typing - - - name: Analysing the code with our lint - working-directory: ${{ inputs.working-directory }} - env: - BLACK_CACHE_DIR: .black_cache - run: | - make lint diff --git a/.github/workflows/build_deploy_image.yml b/.github/workflows/build_deploy_image.yml deleted file mode 100644 index 5550aaab..00000000 --- a/.github/workflows/build_deploy_image.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build, Push, and Deploy Open GPTS - -on: - push: - branches: [main] - workflow_dispatch: - -jobs: - build-and-push: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Short Hash - run: | - echo "GIT_SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - - name: Set up depot.dev multi-arch runner - uses: depot/setup-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.LANGCHAIN_DOCKERHUB_USERNAME }} - password: ${{ secrets.LANGCHAIN_DOCKERHUB_PASSWORD }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push - uses: docker/build-push-action@v5 - with: - push: true - platforms: linux/amd64,linux/arm64 - tags: "docker.io/langchain/open-gpts:${{ env.GIT_SHORT_SHA }}, docker.io/langchain/open-gpts:latest" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 8216a533..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,105 +0,0 @@ ---- -name: CI - -on: - push: - branches: [main] - pull_request: # Trigger on all PRs, ensuring required actions to be run. - workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI - -# If another push to the same PR or branch happens while this workflow is still running, -# cancel the earlier run in favor of the next run. -# -# There's no point in testing an outdated version of the code. GitHub only allows -# a limited number of job runners to be active at the same time, so it's better to cancel -# pointless jobs early so that more useful jobs can run sooner. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - POETRY_VERSION: "1.5.1" - WORKDIR: "./backend" - -jobs: - lint: - uses: ./.github/workflows/_lint.yml - with: - working-directory: "./backend" - secrets: inherit - - test: - timeout-minutes: 5 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ env.WORKDIR }} - strategy: - matrix: - python-version: - - "3.9" - - "3.10" - - "3.11" - name: Python ${{ matrix.python-version }} tests - services: - # Label used to access the service container - postgres: - image: pgvector/pgvector:pg16 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - "5432:5432" - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} - uses: "./.github/actions/poetry_setup" - with: - python-version: ${{ matrix.python-version }} - poetry-version: ${{ env.POETRY_VERSION }} - working-directory: . - cache-key: langserve-all - - name: Install dependencies - run: | - poetry install --with test - - name: Install golang-migrate - run: | - wget -O golang-migrate.deb https://github.com/golang-migrate/migrate/releases/download/v4.17.0/migrate.linux-amd64.deb - sudo dpkg -i golang-migrate.deb && rm golang-migrate.deb - - name: Run tests - env: - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 - POSTGRES_DB: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - SCARF_NO_ANALYTICS: true - run: make test - - frontend-lint-and-build: - runs-on: ubuntu-latest - needs: [lint, test] - steps: - - uses: actions/checkout@v3 - - name: Setup Node.js (LTS) - uses: actions/setup-node@v3 - with: - node-version: '20' - cache: 'yarn' - cache-dependency-path: frontend/yarn.lock - - name: Install frontend dependencies - run: yarn install - working-directory: ./frontend - - name: Run frontend lint - run: yarn lint - working-directory: ./frontend - - name: Build frontend - run: yarn build - working-directory: ./frontend diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml new file mode 100644 index 00000000..5549b03a --- /dev/null +++ b/.github/workflows/deploy-dev.yml @@ -0,0 +1,49 @@ +name: Pulumi Deploy +on: + push: + branches: + - develop +jobs: + preview: + name: Preview + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ secrets.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + # Retrieve and set environment variables from AWS Secrets Manager + - name: Retrieve secrets from AWS Secrets Manager + run: | + aws secretsmanager get-secret-value --secret-id opengpts-env-variables --query 'SecretString' --output text | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' > .env + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # Create and activate the virtual environment + - name: Set up Python virtual environment + run: | + python -m venv .venv + source .venv/bin/activate + python -m pip install --upgrade pip + pip install -r requirements.txt + + # Update the Pulumi stack + - uses: pulumi/actions@v5 + with: + command: up + stack-name: brighthive/bb-assistants-dev + env: + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} diff --git a/.github/workflows/preview-dev.yml b/.github/workflows/preview-dev.yml new file mode 100644 index 00000000..a0eb8a01 --- /dev/null +++ b/.github/workflows/preview-dev.yml @@ -0,0 +1,49 @@ +name: Pulumi Preview +on: + pull_request: + branches: + - develop + +jobs: + preview: + name: Preview + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ secrets.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + # Retrieve and set environment variables from AWS Secrets Manager + - name: Retrieve secrets from AWS Secrets Manager + run: | + aws secretsmanager get-secret-value --secret-id opengpts-env-variables --query 'SecretString' --output text | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' > .env + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # Create and activate the virtual environment + - name: Set up Python virtual environment + run: | + python -m venv .venv + source .venv/bin/activate + python -m pip install --upgrade pip + pip install -r requirements.txt + + - uses: pulumi/actions@v5 + with: + command: preview + stack-name: brighthive/bb-assistants-dev + env: + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} diff --git a/.gitignore b/.gitignore index 23d9237f..a36059a7 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ pnpm-debug.log* # Temp Lambda files: backend/lambda_* lambda_*/ + +# Secrets files +.secrets diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..fdcbc618 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pulumi>=3.0.0,<4.0.0 +pulumi-random>=4.0.0,<5.0.0 +pulumi-aws +pulumi-docker +python-dotenv \ No newline at end of file