Merge pull request #3 from ttngu207/main

update GHA
dj-sciops · Dec 11, 2024 · bd69b0c · bd69b0c
2 parents 044a578 + c8b92cd
commit bd69b0c
Show file tree

Hide file tree

Showing 4 changed files with 288 additions and 9 deletions.
diff --git a/.github/workflows/codebook-profile-build-caller.yaml b/.github/workflows/codebook-profile-build-caller.yaml
@@ -1,4 +1,4 @@
-name: codebook-profile-build
+name: codebook-profile-build-caller
 on:
   workflow_dispatch:
     inputs:
@@ -9,7 +9,7 @@ on:
       py_ver:
         description: "Python version"
         required: true
-        default: "3.10"
+        default: "3.11"
       worker_type:
         description: "Use worker image as codebook profile image"
         required: true
@@ -32,7 +32,7 @@ jobs:
       workflow_version: ${{steps.get_version.outputs.workflow_version}}
   call_sciops_docker_image_debian:
     needs: [get_meta]
-    uses: dj-sciops/.github/.github/workflows/codebook-profile-build-1.0.0.yaml@main
+    uses: ./.github/workflows/codebook-profile-build.yaml
     # uses: yambottle/dj-sciops.github/.github/workflows/codebook-profile-build.yaml@main
     with:
       jhub_ver: ${{ inputs.jhub_ver }}

diff --git a/.github/workflows/codebook-profile-build.yaml b/.github/workflows/codebook-profile-build.yaml
@@ -0,0 +1,127 @@
+name: codebook-profile-build
+on:
+  workflow_call:
+    inputs:
+      jhub_ver:
+        required: true
+        type: string
+      py_ver:
+        required: true
+        type: string
+      worker_type:
+        required: true
+        type: string
+      workflow_version:
+        required: true
+        type: string
+    secrets:
+      RUNNER_PAT:
+        required: true
+      RUNNER_AWS_ACCESS_KEY_ID:
+        required: true
+      RUNNER_AWS_SECRET_ACCESS_KEY:
+        required: true
+      DEPLOY_SSH_KEY_BASE64:
+        required: true
+      DOCKER_REGISTRY_HOST:
+        required: true
+      DOCKER_REGISTRY_REPO:
+        required: true
+      DOCKER_REGISTRY_USERNAME:
+        required: true
+      DOCKER_REGISTRY_PASSWORD:
+        required: true
+jobs:
+  start-runner:
+    timeout-minutes: 5              # normally it only takes 1-2 minutes
+    name: Start self-hosted EC2 runner   
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write        
+    steps:      
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: NextChapterSoftware/ec2-action-builder@main
+        with:
+          github_token: ${{ secrets.RUNNER_PAT}}
+          aws_access_key_id: ${{ secrets.RUNNER_AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-east-2"
+          ec2_instance_type: t3a.medium
+          ec2_ami_id: ami-0172431f1b2be4fc6
+          ec2_subnet_id: "subnet-0a602071414b8b2eb"
+          ec2_security_group_id: "sg-02810c38b0dcb2462"
+          ec2_instance_ttl: 60                # Optional (default is 60 minutes)
+          ec2_spot_instance_strategy: BestEffort    # Other options are: SpotOnly, BestEffort, MaxPerformance 
+          ec2_instance_tags: >
+            [
+              {"Key": "Scope", "Value": "Works"}, 
+              {"Key": "Contract", "Value": "${{github.event.repository.name}}"}, 
+              {"Key": "Application", "Value": "codebook-profile-build-runner"},
+              {"Key": "WorkflowVersion", "Value": "${{ inputs.workflow_version }}"}, 
+              {"Key": "WorkerType", "Value": "${{ inputs.worker_type }}"}
+            ]
+  ## Build/Publish codebook env image
+  build_codebook_image:
+    needs:
+      - start-runner
+    runs-on: ${{ github.run_id }}  
+    env:
+      JHUB_VER: ${{inputs.jhub_ver}}
+      PY_VER: ${{inputs.py_ver}}
+      WORKER_TYPE: ${{inputs.worker_type}}
+      WORKFLOW_VERSION: ${{inputs.workflow_version}}
+    steps:
+      - name: Checkout workflow repo
+        uses: actions/checkout@v4
+      - id: build_env
+        name: Build codebook env image
+        run: |
+          ## Get build info
+          cd ${GITHUB_WORKSPACE}/docker/${WORKER_TYPE}_worker/dist/debian
+          export REPO_NAME=${{github.event.repository.name}}
+          export REPO_OWNER=${{github.repository_owner}}
+          export REPO_BRANCH=${{github.ref_name}}
+
+          echo "${{secrets.DEPLOY_SSH_KEY_BASE64}}" | base64 -di >> ${REPO_NAME}-deploy.pem
+          chmod u=r,g-rwx,o-rwx ${REPO_NAME}-deploy.pem
+          export DEPLOY_KEY=${REPO_NAME}-deploy.pem
+
+          cat <<EOF > docker-compose-codebook-profile.yaml
+          services:
+            codebook_env:
+              build:
+                # only necessary if rebuilding image
+                context: .
+                dockerfile: codebook.Dockerfile
+                args:
+                  - JHUB_VER
+                  - PY_VER
+                  - DEPLOY_KEY
+                  - REPO_OWNER
+                  - REPO_NAME
+                  - REPO_BRANCH
+              image: ${{ secrets.DOCKER_REGISTRY_HOST }}/${{ secrets.DOCKER_REGISTRY_REPO }}/codebook_${REPO_NAME}:singleuser-${JHUB_VER}-py${PY_VER}-${WORKFLOW_VERSION}
+          EOF
+
+          cp ${WORKER_TYPE}_worker.Dockerfile codebook.Dockerfile
+          sed -i '1s/^/ARG JHUB_VER\n/' codebook.Dockerfile
+          sed -i 's@FROM.*@FROM datajoint/djlabhub:singleuser-${JHUB_VER}-py${PY_VER}-latest@g' codebook.Dockerfile
+
+          ## Build image
+          docker compose -f docker-compose-codebook-profile.yaml build
+      - name: Login to vathes Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.DOCKER_REGISTRY_HOST }}
+          username: ${{ secrets.DOCKER_REGISTRY_USERNAME }}
+          password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
+      - name: Push codebook image
+        run: |
+          export IMAGE=${{ secrets.DOCKER_REGISTRY_HOST }}/${{ secrets.DOCKER_REGISTRY_REPO }}/codebook_${{github.event.repository.name}}
+          export TAG=singleuser-${{env.JHUB_VER}}-py${{env.PY_VER}}-${{env.WORKFLOW_VERSION}}
+          echo ${IMAGE} ${TAG}
+          docker push "${IMAGE}:${TAG}"
+          echo ${IMAGE} latest
+          docker tag "${IMAGE}:${TAG}" "${IMAGE}:latest"
+          docker push "${IMAGE}:latest"
diff --git a/.github/workflows/worker-ami-build-1.0.0.yaml b/.github/workflows/worker-ami-build-1.0.0.yaml
@@ -0,0 +1,155 @@
+name: worker-ami-build
+on:
+  workflow_call:
+    inputs:
+      worker_type:
+        required: true
+        type: string
+      workflow_version:
+        required: true
+        type: string
+      staging:
+        required: true
+        type: string
+      runner_timeout:
+        required: false
+        type: number
+        default: 180
+    secrets:
+      RUNNER_PAT:
+        required: true
+      RUNNER_REGION:
+        required: true
+      RUNNER_TYPE:
+        required: true
+      RUNNER_AMI_ID:
+        required: true
+      RUNNER_SUBNET_ID:
+        required: true
+      RUNNER_SG_ID:
+        required: true
+      RUNNER_AWS_ACCESS_KEY_ID:
+        required: true
+      RUNNER_AWS_SECRET_ACCESS_KEY:
+        required: true
+      DJ_HOST:
+        required: true
+      DJ_USER:
+        required: true
+      DJ_PASS:
+        required: true
+      BUILD_PAT:
+        required: true
+      BUILD_AWS_ACCESS_KEY_ID:
+        required: true
+      BUILD_AWS_SECRET_ACCESS_KEY:
+        required: true
+      DEPLOY_SSH_KEY_BASE64:
+        required: true
+      DOCKER_REGISTRY_HOST:
+        required: true
+      DOCKER_REGISTRY_REPO:
+        required: true
+      DOCKER_REGISTRY_USERNAME:
+        required: true
+      DOCKER_REGISTRY_PASSWORD:
+        required: true
+
+jobs:
+  start-runner:
+    timeout-minutes: 5              # normally it only takes 1-2 minutes
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    steps:
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: NextChapterSoftware/ec2-action-builder@main
+        with:
+          github_token: ${{ secrets.RUNNER_PAT }}
+          aws_access_key_id: ${{ secrets.RUNNER_AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws_region: ${{ secrets.RUNNER_REGION }}
+          ec2_instance_type: ${{ secrets.RUNNER_TYPE }}
+          ec2_ami_id: ${{ secrets.RUNNER_AMI_ID }}
+          ec2_subnet_id: ${{ secrets.RUNNER_SUBNET_ID }}
+          ec2_security_group_id: ${{ secrets.RUNNER_SG_ID }}
+          ec2_instance_ttl: ${{ inputs.runner_timeout }}  # Optional (default is 60 minutes)
+          ec2_spot_instance_strategy: BestEffort    # Other options are: SpotOnly, BestEffort, MaxPerformance
+          ec2_instance_tags: >
+            [
+              {"Key": "Scope", "Value": "Works"}, 
+              {"Key": "Contract", "Value": "${{github.event.repository.name}}"}, 
+              {"Key": "Application", "Value": "worker-ami-build-runner"},
+              {"Key": "WorkflowVersion", "Value": "${{ inputs.workflow_version }}"}, 
+              {"Key": "WorkerType", "Value": "${{ inputs.worker_type }}"}, 
+              {"Key": "Staging", "Value": "${{ inputs.staging }}"}
+            ]
+  build_worker_ami:
+    needs:
+      - start-runner
+    runs-on: ${{ github.run_id }}
+    env:
+      HOME: /root
+    steps:
+      - name: Checkout packer build repo
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BUILD_PAT }}
+          repository: 'datajoint-company/dj-gitops'
+          path: 'dj-gitops'
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        # WARNING: Running pip as the 'root'
+        # https://github.com/actions/setup-python/issues/513
+        with:
+          python-version: '3.10'
+      - run: pip install --user datajoint
+      - name: Call fetcher
+        env:
+          DJ_HOST: ${{ secrets.DJ_HOST }}
+          DJ_USER: ${{ secrets.DJ_USER }}
+          DJ_PASS: ${{ secrets.DJ_PASS }}
+          REPO_NAME: ${{github.event.repository.name}}
+          WORKER_TYPE: ${{ inputs.worker_type }}
+          WORKFLOW_VERSION: ${{ inputs.workflow_version }}
+          STAGING: ${{ inputs.staging }}
+        run: |
+          export ORG_NAME=$(echo ${REPO_NAME} | cut -d "_" -f 1)
+          export WORKFLOW_NAME=$(echo ${REPO_NAME} | cut -d "_" -f 2)
+          cd ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami/inputs/
+          python fetcher.py
+      - name: Build worker AMI
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.BUILD_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.BUILD_AWS_SECRET_ACCESS_KEY }}
+          STAGING: ${{ inputs.staging }}
+          REPO_NAME: ${{github.event.repository.name}}
+          DOCKER_REGISTRY_HOST: ${{ secrets.DOCKER_REGISTRY_HOST }}
+          DOCKER_REGISTRY_REPO: ${{ secrets.DOCKER_REGISTRY_REPO }}
+          DOCKER_REGISTRY_USERNAME: ${{ secrets.DOCKER_REGISTRY_USERNAME }}
+          DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
+        run: |
+          # Set up deploy SSH key
+          echo -n "${{ secrets.DEPLOY_SSH_KEY_BASE64 }}" | base64 -d > ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami/keys/${REPO_NAME}-deploy.pem
+          cd ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami
+          packer init .
+          packer build -var-file ./inputs/${REPO_NAME}/${STAGING}.pkrvars.hcl . | tee ./outputs/packer.temp.log
+      - name: Terminate builder
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.BUILD_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.BUILD_AWS_SECRET_ACCESS_KEY }}
+        if: always()
+        run: |
+          cd ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami/outputs
+          bash terminate_builder_by_log.sh packer.temp.log
+      - name: Update worker AMI metadata
+        if: ${{ inputs.staging == 'stable' }}
+        env:
+          DJ_HOST: ${{ secrets.DJ_HOST }}
+          DJ_USER: ${{ secrets.DJ_USER }}
+          DJ_PASS: ${{ secrets.DJ_PASS }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami/outputs/
+          python insert_ami_meta.py
diff --git a/.github/workflows/worker-build-caller.yaml b/.github/workflows/worker-build-caller.yaml
@@ -1,4 +1,4 @@
-name: worker-build
+name: worker-build-caller
 on:
   workflow_dispatch:
     inputs:
@@ -8,9 +8,6 @@ on:
         type: choice
         options:
           - standard
-          - spike_processing
-          - spike_sorting
-          - dlc
       staging:
         description: "Staging"
         required: true
@@ -35,8 +32,8 @@ jobs:
       workflow_version: ${{steps.get_version.outputs.workflow_version}}
   call-worker-ami-build:
     needs: [get_meta]
-    uses: dj-sciops/.github/.github/workflows/worker-ami-build-1.0.0.yaml@main
-    # uses: yambottle/dj-sciops.github/.github/workflows/worker-ami-build.yaml@main
+    uses: ./.github/workflows/worker-ami-build-1.0.0.yaml
+    # uses: yambottle/dj-sciops.github/.github/workflows/worker-ami-build-1.0.0.yaml@main
     with:
       worker_type: ${{github.event.inputs.worker_type}}
       workflow_version: ${{needs.get_meta.outputs.workflow_version}}