diff --git a/.github/workflows/codebook-profile-build-caller.yaml b/.github/workflows/codebook-profile-build-caller.yaml index 862dcd0..00eff8d 100644 --- a/.github/workflows/codebook-profile-build-caller.yaml +++ b/.github/workflows/codebook-profile-build-caller.yaml @@ -1,4 +1,4 @@ -name: codebook-profile-build +name: codebook-profile-build-caller on: workflow_dispatch: inputs: @@ -9,7 +9,7 @@ on: py_ver: description: "Python version" required: true - default: "3.10" + default: "3.11" worker_type: description: "Use worker image as codebook profile image" required: true @@ -32,7 +32,7 @@ jobs: workflow_version: ${{steps.get_version.outputs.workflow_version}} call_sciops_docker_image_debian: needs: [get_meta] - uses: dj-sciops/.github/.github/workflows/codebook-profile-build-1.0.0.yaml@main + uses: ./.github/workflows/codebook-profile-build.yaml # uses: yambottle/dj-sciops.github/.github/workflows/codebook-profile-build.yaml@main with: jhub_ver: ${{ inputs.jhub_ver }} diff --git a/.github/workflows/codebook-profile-build.yaml b/.github/workflows/codebook-profile-build.yaml new file mode 100644 index 0000000..281941e --- /dev/null +++ b/.github/workflows/codebook-profile-build.yaml @@ -0,0 +1,127 @@ +name: codebook-profile-build +on: + workflow_call: + inputs: + jhub_ver: + required: true + type: string + py_ver: + required: true + type: string + worker_type: + required: true + type: string + workflow_version: + required: true + type: string + secrets: + RUNNER_PAT: + required: true + RUNNER_AWS_ACCESS_KEY_ID: + required: true + RUNNER_AWS_SECRET_ACCESS_KEY: + required: true + DEPLOY_SSH_KEY_BASE64: + required: true + DOCKER_REGISTRY_HOST: + required: true + DOCKER_REGISTRY_REPO: + required: true + DOCKER_REGISTRY_USERNAME: + required: true + DOCKER_REGISTRY_PASSWORD: + required: true +jobs: + start-runner: + timeout-minutes: 5 # normally it only takes 1-2 minutes + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: NextChapterSoftware/ec2-action-builder@main + with: + github_token: ${{ secrets.RUNNER_PAT}} + aws_access_key_id: ${{ secrets.RUNNER_AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.RUNNER_AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + ec2_instance_type: t3a.medium + ec2_ami_id: ami-0172431f1b2be4fc6 + ec2_subnet_id: "subnet-0a602071414b8b2eb" + ec2_security_group_id: "sg-02810c38b0dcb2462" + ec2_instance_ttl: 60 # Optional (default is 60 minutes) + ec2_spot_instance_strategy: BestEffort # Other options are: SpotOnly, BestEffort, MaxPerformance + ec2_instance_tags: > + [ + {"Key": "Scope", "Value": "Works"}, + {"Key": "Contract", "Value": "${{github.event.repository.name}}"}, + {"Key": "Application", "Value": "codebook-profile-build-runner"}, + {"Key": "WorkflowVersion", "Value": "${{ inputs.workflow_version }}"}, + {"Key": "WorkerType", "Value": "${{ inputs.worker_type }}"} + ] + ## Build/Publish codebook env image + build_codebook_image: + needs: + - start-runner + runs-on: ${{ github.run_id }} + env: + JHUB_VER: ${{inputs.jhub_ver}} + PY_VER: ${{inputs.py_ver}} + WORKER_TYPE: ${{inputs.worker_type}} + WORKFLOW_VERSION: ${{inputs.workflow_version}} + steps: + - name: Checkout workflow repo + uses: actions/checkout@v4 + - id: build_env + name: Build codebook env image + run: | + ## Get build info + cd ${GITHUB_WORKSPACE}/docker/${WORKER_TYPE}_worker/dist/debian + export REPO_NAME=${{github.event.repository.name}} + export REPO_OWNER=${{github.repository_owner}} + export REPO_BRANCH=${{github.ref_name}} + + echo "${{secrets.DEPLOY_SSH_KEY_BASE64}}" | base64 -di >> ${REPO_NAME}-deploy.pem + chmod u=r,g-rwx,o-rwx ${REPO_NAME}-deploy.pem + export DEPLOY_KEY=${REPO_NAME}-deploy.pem + + cat < docker-compose-codebook-profile.yaml + services: + codebook_env: + build: + # only necessary if rebuilding image + context: . + dockerfile: codebook.Dockerfile + args: + - JHUB_VER + - PY_VER + - DEPLOY_KEY + - REPO_OWNER + - REPO_NAME + - REPO_BRANCH + image: ${{ secrets.DOCKER_REGISTRY_HOST }}/${{ secrets.DOCKER_REGISTRY_REPO }}/codebook_${REPO_NAME}:singleuser-${JHUB_VER}-py${PY_VER}-${WORKFLOW_VERSION} + EOF + + cp ${WORKER_TYPE}_worker.Dockerfile codebook.Dockerfile + sed -i '1s/^/ARG JHUB_VER\n/' codebook.Dockerfile + sed -i 's@FROM.*@FROM datajoint/djlabhub:singleuser-${JHUB_VER}-py${PY_VER}-latest@g' codebook.Dockerfile + + ## Build image + docker compose -f docker-compose-codebook-profile.yaml build + - name: Login to vathes Registry + uses: docker/login-action@v3 + with: + registry: ${{ secrets.DOCKER_REGISTRY_HOST }} + username: ${{ secrets.DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + - name: Push codebook image + run: | + export IMAGE=${{ secrets.DOCKER_REGISTRY_HOST }}/${{ secrets.DOCKER_REGISTRY_REPO }}/codebook_${{github.event.repository.name}} + export TAG=singleuser-${{env.JHUB_VER}}-py${{env.PY_VER}}-${{env.WORKFLOW_VERSION}} + echo ${IMAGE} ${TAG} + docker push "${IMAGE}:${TAG}" + echo ${IMAGE} latest + docker tag "${IMAGE}:${TAG}" "${IMAGE}:latest" + docker push "${IMAGE}:latest" \ No newline at end of file diff --git a/.github/workflows/worker-ami-build-1.0.0.yaml b/.github/workflows/worker-ami-build-1.0.0.yaml new file mode 100644 index 0000000..b65cc4d --- /dev/null +++ b/.github/workflows/worker-ami-build-1.0.0.yaml @@ -0,0 +1,155 @@ +name: worker-ami-build +on: + workflow_call: + inputs: + worker_type: + required: true + type: string + workflow_version: + required: true + type: string + staging: + required: true + type: string + runner_timeout: + required: false + type: number + default: 180 + secrets: + RUNNER_PAT: + required: true + RUNNER_REGION: + required: true + RUNNER_TYPE: + required: true + RUNNER_AMI_ID: + required: true + RUNNER_SUBNET_ID: + required: true + RUNNER_SG_ID: + required: true + RUNNER_AWS_ACCESS_KEY_ID: + required: true + RUNNER_AWS_SECRET_ACCESS_KEY: + required: true + DJ_HOST: + required: true + DJ_USER: + required: true + DJ_PASS: + required: true + BUILD_PAT: + required: true + BUILD_AWS_ACCESS_KEY_ID: + required: true + BUILD_AWS_SECRET_ACCESS_KEY: + required: true + DEPLOY_SSH_KEY_BASE64: + required: true + DOCKER_REGISTRY_HOST: + required: true + DOCKER_REGISTRY_REPO: + required: true + DOCKER_REGISTRY_USERNAME: + required: true + DOCKER_REGISTRY_PASSWORD: + required: true + +jobs: + start-runner: + timeout-minutes: 5 # normally it only takes 1-2 minutes + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: NextChapterSoftware/ec2-action-builder@main + with: + github_token: ${{ secrets.RUNNER_PAT }} + aws_access_key_id: ${{ secrets.RUNNER_AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.RUNNER_AWS_SECRET_ACCESS_KEY }} + aws_region: ${{ secrets.RUNNER_REGION }} + ec2_instance_type: ${{ secrets.RUNNER_TYPE }} + ec2_ami_id: ${{ secrets.RUNNER_AMI_ID }} + ec2_subnet_id: ${{ secrets.RUNNER_SUBNET_ID }} + ec2_security_group_id: ${{ secrets.RUNNER_SG_ID }} + ec2_instance_ttl: ${{ inputs.runner_timeout }} # Optional (default is 60 minutes) + ec2_spot_instance_strategy: BestEffort # Other options are: SpotOnly, BestEffort, MaxPerformance + ec2_instance_tags: > + [ + {"Key": "Scope", "Value": "Works"}, + {"Key": "Contract", "Value": "${{github.event.repository.name}}"}, + {"Key": "Application", "Value": "worker-ami-build-runner"}, + {"Key": "WorkflowVersion", "Value": "${{ inputs.workflow_version }}"}, + {"Key": "WorkerType", "Value": "${{ inputs.worker_type }}"}, + {"Key": "Staging", "Value": "${{ inputs.staging }}"} + ] + build_worker_ami: + needs: + - start-runner + runs-on: ${{ github.run_id }} + env: + HOME: /root + steps: + - name: Checkout packer build repo + uses: actions/checkout@v4 + with: + token: ${{ secrets.BUILD_PAT }} + repository: 'datajoint-company/dj-gitops' + path: 'dj-gitops' + - name: Setup Python + uses: actions/setup-python@v5 + # WARNING: Running pip as the 'root' + # https://github.com/actions/setup-python/issues/513 + with: + python-version: '3.10' + - run: pip install --user datajoint + - name: Call fetcher + env: + DJ_HOST: ${{ secrets.DJ_HOST }} + DJ_USER: ${{ secrets.DJ_USER }} + DJ_PASS: ${{ secrets.DJ_PASS }} + REPO_NAME: ${{github.event.repository.name}} + WORKER_TYPE: ${{ inputs.worker_type }} + WORKFLOW_VERSION: ${{ inputs.workflow_version }} + STAGING: ${{ inputs.staging }} + run: | + export ORG_NAME=$(echo ${REPO_NAME} | cut -d "_" -f 1) + export WORKFLOW_NAME=$(echo ${REPO_NAME} | cut -d "_" -f 2) + cd ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami/inputs/ + python fetcher.py + - name: Build worker AMI + env: + AWS_ACCESS_KEY_ID: ${{ secrets.BUILD_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.BUILD_AWS_SECRET_ACCESS_KEY }} + STAGING: ${{ inputs.staging }} + REPO_NAME: ${{github.event.repository.name}} + DOCKER_REGISTRY_HOST: ${{ secrets.DOCKER_REGISTRY_HOST }} + DOCKER_REGISTRY_REPO: ${{ secrets.DOCKER_REGISTRY_REPO }} + DOCKER_REGISTRY_USERNAME: ${{ secrets.DOCKER_REGISTRY_USERNAME }} + DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + run: | + # Set up deploy SSH key + echo -n "${{ secrets.DEPLOY_SSH_KEY_BASE64 }}" | base64 -d > ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami/keys/${REPO_NAME}-deploy.pem + cd ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami + packer init . + packer build -var-file ./inputs/${REPO_NAME}/${STAGING}.pkrvars.hcl . | tee ./outputs/packer.temp.log + - name: Terminate builder + env: + AWS_ACCESS_KEY_ID: ${{ secrets.BUILD_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.BUILD_AWS_SECRET_ACCESS_KEY }} + if: always() + run: | + cd ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami/outputs + bash terminate_builder_by_log.sh packer.temp.log + - name: Update worker AMI metadata + if: ${{ inputs.staging == 'stable' }} + env: + DJ_HOST: ${{ secrets.DJ_HOST }} + DJ_USER: ${{ secrets.DJ_USER }} + DJ_PASS: ${{ secrets.DJ_PASS }} + run: | + cd ${GITHUB_WORKSPACE}/dj-gitops/infrastructures/packer/worker_ami/outputs/ + python insert_ami_meta.py diff --git a/.github/workflows/worker-build-caller.yaml b/.github/workflows/worker-build-caller.yaml index 56fbb89..63cd7b2 100644 --- a/.github/workflows/worker-build-caller.yaml +++ b/.github/workflows/worker-build-caller.yaml @@ -1,4 +1,4 @@ -name: worker-build +name: worker-build-caller on: workflow_dispatch: inputs: @@ -8,9 +8,6 @@ on: type: choice options: - standard - - spike_processing - - spike_sorting - - dlc staging: description: "Staging" required: true @@ -35,8 +32,8 @@ jobs: workflow_version: ${{steps.get_version.outputs.workflow_version}} call-worker-ami-build: needs: [get_meta] - uses: dj-sciops/.github/.github/workflows/worker-ami-build-1.0.0.yaml@main - # uses: yambottle/dj-sciops.github/.github/workflows/worker-ami-build.yaml@main + uses: ./.github/workflows/worker-ami-build-1.0.0.yaml + # uses: yambottle/dj-sciops.github/.github/workflows/worker-ami-build-1.0.0.yaml@main with: worker_type: ${{github.event.inputs.worker_type}} workflow_version: ${{needs.get_meta.outputs.workflow_version}}