#502 Moved ci to AWS code build (#275)

Related to exasol/script-languages-release#502

.github/workflows/check_aws_buildspec.yaml

@@ -0,0 +1,36 @@
+name: Check AWS buildspec
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Init submodules
+      run: git submodule update --init --recursive
+
+    - name: Setup Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install Poetry
+      uses: abatilo/[email protected]
+      with:
+        poetry-version: 1.1.13
+
+    - name: Setup poetry env
+      run:  bash scripts/setup_poetry_env.sh "python3.8"
+
+    - name: Run githooks/update_aws_buildspec.sh
+      run: bash githooks/update_aws_buildspec.sh
+
+    - name: Check git status
+      run: bash scripts/check_git_status.sh

.github/workflows/check_bazel_tests.yml

@@ -2,8 +2,11 @@ name: Check Bazel Tests
 
 on:
   push:
+    branches:
+      - master
   pull_request:
 
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -33,4 +36,4 @@ jobs:
     - name: Test
       run: |
         bazel test //javacontainer/test/...
-      working-directory: ./exaudfclient/base
+      working-directory: ./exaudfclient/base

.github/workflows/check_error_codes.yml

@@ -3,10 +3,9 @@ name: Check error codes
 on:
   push:
     branches:
-      - "*"
+      - master
   pull_request:
-    branches: 
-      - "*"
+
 
 jobs:
   build:
@@ -20,4 +19,3 @@ jobs:
 
     - name: Search for duplicated error codes
       run: bash find_duplicate_error_codes.sh
-

.github/workflows/check_if_submodules_head_are_ancestors_of_origin_master.yaml

@@ -3,10 +3,9 @@ name: Check if submodules heads are ancestors of origin/master
 on:
   push:
     branches:
-      - "*"
+      - master
   pull_request:
-    branches: 
-      - "*"
+
 
 jobs:
   build:

.github/workflows/check_markdown_links.yaml

@@ -1,6 +1,11 @@
 name: Check Markdown links
 
-on: push
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
 
 jobs:
   markdown-link-check:

.github/workflows/check_starter_scripts.yaml

@@ -0,0 +1,36 @@
+name: Check starter scripts
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Init submodules
+      run: git submodule update --init --recursive
+
+    - name: Setup Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install Poetry
+      uses: abatilo/[email protected]
+      with:
+        poetry-version: 1.1.13
+
+    - name: Setup poetry env
+      run:  bash scripts/setup_poetry_env.sh "python3.8"
+
+    - name: Run githooks/update_exaslct_starter_scripts.sh
+      run: bash githooks/update_exaslct_starter_scripts.sh
+
+    - name: Check git status
+      run: bash scripts/check_git_status.sh

.github/workflows/shellcheck.yaml

@@ -1,6 +1,10 @@
 name: Check bash scripts
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 
 jobs:
   shellcheck:

.github/workflows/test_package_management_scripts.yaml

@@ -3,10 +3,8 @@ name: Test package management scripts
 on:
   push:
     branches:
-      - "*"
+      - master
   pull_request:
-    branches: 
-      - "*"
 
 jobs:
   test_package_management_scripts:

.gitignore

@@ -11,4 +11,6 @@ emulator/zmqcontainer_pb2.py
 .build_output/
 luigi.cfg
 *.iml
-exaudfclient/base/file.txt
+/codebuild_build.sh
+exaudfclient/base/file.txt
+dist

README.md

@@ -1,3 +1,3 @@
 # EXASOL Script Languages Base Repository
 
-**Please note, that this is no longer the main repository of the script-languges. Please visit the new main repository [exasol/script-languages-release](https://github.com/exasol/script-languages-release). If you find issues or have feature requests, please report them on the [issue page of the main repository](https://github.com/exasol/script-languages-release/issues) or as an open source project which is officially supported by EXASOL, you can contact our support team, as well.**
+**Please note, that this is no longer the main repository of the script-languages. Please visit the new main repository [exasol/script-languages-release](https://github.com/exasol/script-languages-release). If you find issues or have feature requests, please report them on the [issue page of the main repository](https://github.com/exasol/script-languages-release/issues) or as an open source project which is officially supported by EXASOL, you can contact our support team, as well.**

aws-code-build/DeveloperGuide.md

@@ -0,0 +1,49 @@
+## Local Run
+
+AWS Code Build provides docker containers and a small script to run single builds locally.
+Note: This does not work for batch builds, only single builds.
+Another limitation is, that artifacts will not be uploaded to S3.
+
+### Requirements
+- Docker
+- AWS CLI setup
+- GIT
+
+
+### Setup
+
+[Here](https://docs.aws.amazon.com/codebuild/latest/userguide/use-codebuild-agent.html) are detailed scripts for the installation.
+
+There are two options:
+1. Build the AWS build container from scratch which might take several hours. You can use the script [get_from_source.sh](run_local/get_from_source.sh) which runs the installation automatically.
+2. Use the prebuilt AWS build container from Dockerhub in the `exadockerci4/aws_codebuild` repository (which might not be up-tp-date). You can use the script [get_prebuilt.sh](run_local/get_prebuilt.sh) which runs the installation automatically.
+
+### Run
+
+- Set the flavor in aws-code-build/run_local_run_local.env
+- Run [run_local.sh](run_local/run_local.sh) or [run_local_prebuilt.sh](run_local/run_local_prebuilt.sh) with buildspec file and AWS profile as parameters:
+
+```bash
+bash ./aws-code-build/run_local/run_local.sh aws-code-build/ci/build_buildspec.yaml my_aws_profile_mfa
+```
+
+## Buildspec Updates
+
+The buildspec files are automatically generated by `script-languages-container-ci-setup`. There is a [githook](../githooks/update_aws_buildspec.sh) which ensures that the buildspecs are up-to-date.
+
+## AWS Deployment
+
+The infrastructure is created using AWS Cloudformation. The cloudformation stack can be automatically deployed by using the script [update_aws_build_stack.sh](update_aws_build_stack.sh). You need to pass your AWS profile as parameter, for example:
+
+```bash
+bash ./aws-code-build/update_aws_build_stack.sh my-aws-profile-mfa
+```
+
+## AWS Build Overview
+
+- AWS CodeBuild has registered a WebHook in [script-languages](https://github.com/exasol/script-languages) and [script-languages-release](https://github.com/exasol/script-languages-release), which will trigger AWS builds automatically
+- Access the AWS Console and go to "CodeBuild"
+- On the left side select Build -> Build projects 
+- Select the SLCCodeBuild_... or SLCReleaseCodeBuild_... project
+- In the tag select "Batch History", there you will see all started batch builds for a commit.
+- Select a batch build, and you can see all single builds (for each flavor) which belong to the batch build

aws-code-build/build_config.json

@@ -0,0 +1,9 @@
+{"build_ignore":
+  {"ignored_paths":
+    ["google-cloud-build", "python_client", ".gitignore", ".dockerignore", "LICENSE",
+      "README.md", "find_duplicate_error_codes.sh", "find_error_codes.sh",
+      "find_highest_error_codes_per_module.sh", "find_incomplete_error_codes.sh", "find_next_error_code_per_module.sh",
+      "show_code_arround_error_codes.sh", "visualize_task_dependencies.sh", "udf-script-signature-generator",
+      ".github", "cpp_client", "emulator", "exaslct_scripts", "githooks"]
+  }
+}

aws-code-build/ci/build_buildspec.yaml

@@ -0,0 +1,48 @@
+version: 0.2
+
+# ---- AUTOMATICALLY GENERATED FILE --------
+# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "script-languages-container-ci-setup" TO UPDATE ---
+env:
+  shell: bash
+  variables:
+    BUILD_DOCKER_REPOSITORY: "exadockerci4/script-languages-build-cache"
+    RELEASE_DOCKER_REPOSITORY: "exasol/script-language-container"
+    CUSTOM_BRANCH: ""
+  secrets-manager:
+    DOCKER_USER: "Dockerhub:User"
+    DOCKER_PASSWORD: "Dockerhub:AccessToken"
+phases:
+
+  install:
+    runtime-versions:
+      python: 3.8
+    commands:
+      - git submodule update --init --recursive
+      - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3 -
+      - export PATH=$PATH:$HOME/.poetry/bin
+      - poetry env use $(command -v "python3.8")
+      - poetry install
+
+  pre_build:
+      commands:
+        - echo CODEBUILD_RESOLVED_SOURCE_VERSION is "$CODEBUILD_RESOLVED_SOURCE_VERSION" #supposed to be the SHA
+        - echo CODEBUILD_SOURCE_VERSION is "$CODEBUILD_SOURCE_VERSION" #supposed to be the SHA
+        - "[[ -z \"${CODEBUILD_WEBHOOK_HEAD_REF}\" ]] && export CURRENT_BRANCH=\"$CUSTOM_BRANCH\" || export CURRENT_BRANCH=\"$CODEBUILD_WEBHOOK_HEAD_REF\""
+        - echo CURRENT_BRANCH is "$CURRENT_BRANCH"
+        - echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USER" --password-stdin
+        - mkdir -p ".build_output/jobs" ".build_output/security_scan" ".build_output/metadata" #Create output directory here. Otherwise artifacts phase might fail.
+        - echo "$(date)" > .build_output/metadata/start_date
+        - echo "$CODEBUILD_SOURCE_VERSION" > .build_output/metadata/commit_sha
+        - echo "$CURRENT_BRANCH" > .build_output/metadata/branch
+  build:
+      commands:
+        - poetry run python3 -m exasol_script_languages_container_ci.main run-ci --flavor $FLAVOR --branch-name "$CURRENT_BRANCH" --docker-user "$DOCKER_USER" --docker-password "$DOCKER_PASSWORD" --docker-build-repository "$BUILD_DOCKER_REPOSITORY" --docker-release-repository "$RELEASE_DOCKER_REPOSITORY" --commit-sha "$CODEBUILD_SOURCE_VERSION" --config-file aws-code-build/build_config.json
+
+artifacts:
+  files:
+      - 'jobs/**/*'
+      - 'security_scan/**/*'
+      - 'metadata/**/*'
+  name: build_output_$(date +%Y-%m-%d-%Hh-%Mm-%Ss)
+  base-directory: .build_output
+  s3-prefix: flavor_$FLAVOR

aws-code-build/ci/buildspec.yaml

@@ -0,0 +1,36 @@
+version: 0.2
+
+# ---- AUTOMATICALLY GENERATED FILE --------
+# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "script-languages-container-ci-setup" TO UPDATE ---
+
+batch:
+  fast-fail: false
+  build-graph:
+    - identifier: build_python_36_data_science_cuda_EXASOL_620
+      env:
+        variables:
+          FLAVOR: python-3.6-data-science-cuda-EXASOL-6.2.0
+      buildspec: ./aws-code-build/ci/build_buildspec.yaml
+      privileged-mode: true
+      type: BUILD_GENERAL1_MEDIUM
+    - identifier: build_python_36_minimal_EXASOL_620
+      env:
+        variables:
+          FLAVOR: python-3.6-minimal-EXASOL-6.2.0
+      buildspec: ./aws-code-build/ci/build_buildspec.yaml
+      privileged-mode: true
+      type: BUILD_GENERAL1_MEDIUM
+    - identifier: build_python_37_minimal_EXASOL_620
+      env:
+        variables:
+          FLAVOR: python-3.7-minimal-EXASOL-6.2.0
+      buildspec: ./aws-code-build/ci/build_buildspec.yaml
+      privileged-mode: true
+      type: BUILD_GENERAL1_MEDIUM
+    - identifier: build_python_38_minimal_EXASOL_620
+      env:
+        variables:
+          FLAVOR: python-3.8-minimal-EXASOL-6.2.0
+      buildspec: ./aws-code-build/ci/build_buildspec.yaml
+      privileged-mode: true
+      type: BUILD_GENERAL1_MEDIUM

aws-code-build/run_local/get_from_source.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -euo pipefail
+
+pushd "../.." &>/dev/null
+
+echo Pulling codebuild script...
+wget https://raw.githubusercontent.com/aws/aws-codebuild-docker-images/master/local_builds/codebuild_build.sh
+if [[ ! $(docker image ls aws/codebuild/ubuntu/standard:5.0 --format="true") ]] ;
+then
+ echo "pulling https://github.com/aws/aws-codebuild-docker-images"
+ pushd /tmp
+ git clone https://github.com/aws/aws-codebuild-docker-images
+ pushd aws-codebuild-docker-images/ubuntu/standard/5.0
+ echo "Building aws/codebuild/ubuntu/standard:5.0"
+ docker build -t aws/codebuild/ubuntu/standard:5.0 .
+ popd
+ rm -rf aws-codebuild-docker-images
+ popd
+fi
+
+echo Ready.
+echo Now you can run the AWS code build locally with:
+echo "'./aws-code-build/run_local/run_local_prebuilt.sh <buildspec> <aws-profile> in the root folder"
+
+popd &> /dev/null

aws-code-build/run_local/get_prebuilt.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -euo pipefail
+
+pushd "../.." &>/dev/null
+
+echo Pulling codebuild script...
+wget https://raw.githubusercontent.com/aws/aws-codebuild-docker-images/master/local_builds/codebuild_build.sh
+docker pull exadockerci4/aws_codebuild:5.0
+echo Ready.
+echo Now you can run the AWS code build locally with:
+echo "'./aws-code-build/run_local/run_local_prebuilt.sh <buildspec> <aws-profile> in the root folder"
+
+popd &> /dev/null

aws-code-build/run_local/run_local.env

@@ -0,0 +1,5 @@
+FLAVOR=python-3.8-minimal-EXASOL-6.2.0
+CODEBUILD_SOURCE_VERSION=6a56214f8683be5346ce0cfcebf2016658e1fd1f
+BUILD_DOCKER_REPOSITORY="exatk/script-languages-build-cache"
+RELEASE_DOCKER_REPOSITORY="exasol/script-language-container"
+CUSTOM_BRANCH=refs/heads/master

aws-code-build/run_local/run_local.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+BUILD_SPEC=$1
+AWS_PROFILE=$2
+SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+pushd "$SCRIPT_DIR/../../" &>/dev/null
+#Artifacts will be stored under /tmp
+./codebuild_build.sh -e ./aws-code-build/run_local/run_local.env -d -c -i aws/codebuild/ubuntu/standard:5.0 -b "$BUILD_SPEC" -p $AWS_PROFILE -a /tmp
+popd > /dev/null

aws-code-build/run_local/run_local_prebuilt.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+BUILD_SPEC=$1
+AWS_PROFILE=$2
+SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+pushd "$SCRIPT_DIR/../../" &>/dev/null
+#Artifacts will be stored under /tmp
+./codebuild_build.sh -e ./aws-code-build/run_local/run_local.env -d -c -i exadockerci4/aws_codebuild:5.0 -b "$BUILD_SPEC" -p $AWS_PROFILE -a /tmp
+popd > /dev/null