From 4f2d16676615241c4b91a471ef99e81eeca69214 Mon Sep 17 00:00:00 2001 From: Niklas Koehnecke Date: Tue, 9 Apr 2024 10:42:58 +0200 Subject: [PATCH] refactor: extract current github jobs for reuse --- .github/workflows/daily.yml | 14 ++ .github/workflows/doctest.yml | 39 ++++++ .github/workflows/github-actions.yml | 202 +-------------------------- .github/workflows/lint.yml | 44 ++++++ .github/workflows/run_notebooks.yml | 51 +++++++ .github/workflows/test.yml | 64 +++++++++ 6 files changed, 217 insertions(+), 197 deletions(-) create mode 100644 .github/workflows/daily.yml create mode 100644 .github/workflows/doctest.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/run_notebooks.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml new file mode 100644 index 000000000..5077252e6 --- /dev/null +++ b/.github/workflows/daily.yml @@ -0,0 +1,14 @@ +name: CI + +# Scheduled workflows will only run on the default branch. +on: + schedule: + - cron: '0 0 * * *' # runs once a day at midnight in the timezone of your GitHub repository + +jobs: + build: + runs-on: [windows-latest, macos-latest] + steps: + - name: Checkout code + uses: actions/checkout@v2 + # your steps go here diff --git a/.github/workflows/doctest.yml b/.github/workflows/doctest.yml new file mode 100644 index 000000000..0e87b516d --- /dev/null +++ b/.github/workflows/doctest.yml @@ -0,0 +1,39 @@ +on: + workflow_call: +jobs: + doctest: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + poetry config installer.max-workers 10 + poetry install --no-interaction + - name: install Sphinx + run: sudo apt-get update -y && sudo apt-get install python3-sphinx + - name: run doctest + env: + AA_TOKEN: ${{ secrets.AA_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + CLIENT_URL: "https://api.aleph-alpha.com" + run: ./scripts/doctest.sh diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 8bdddb7f2..6d9f14617 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -9,206 +9,14 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true - jobs: doctest: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry config installer.max-workers 10 - poetry install --no-interaction - - - name: install Sphinx - run: sudo apt-get update -y && sudo apt-get install python3-sphinx - - - name: run doctest - env: - AA_TOKEN: ${{ secrets.AA_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - CLIENT_URL: "https://api.aleph-alpha.com" - run: ./scripts/doctest.sh - + uses: ./.github/workflows/python/doctest.yml lint: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry config installer.max-workers 10 - poetry install --no-interaction - - - name: set PY for pre-commit - run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - - uses: actions/cache@v4 - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - - name: Run linters - run: | - ./scripts/lint.sh - + uses: ./.github/workflows/python/lint.yml test: - runs-on: ubuntu-latest - - services: - argilla-elastic-search: - image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 - env: - ES_JAVA_OPTS: "-Xms512m -Xmx512m" - discovery.type: "single-node" - xpack.security.enabled: "false" - argilla: - image: argilla/argilla-server - ports: - - "6900:6900" - env: - ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200" - open-telemetry-trace-service: - env: - COLLECTOR_OTLP_ENABLED: "true" - ports: - - "4317:4317" - - "4318:4318" - - "16686:16686" - image: jaegertracing/all-in-one:1.35 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry config installer.max-workers 10 - poetry install --no-interaction - - - name: Run pytest - env: - AA_TOKEN: ${{ secrets.AA_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - ARGILLA_API_URL: "http://localhost:6900/" - ARGILLA_API_KEY: "argilla.apikey" - CLIENT_URL: "https://api.aleph-alpha.com" - run: | - ./scripts/test.sh - + uses: ./.github/workflows/python/test.yml run-notebooks: - runs-on: ubuntu-latest - - services: - argilla-elastic-search: - image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 - env: - ES_JAVA_OPTS: "-Xms512m -Xmx512m" - discovery.type: "single-node" - xpack.security.enabled: "false" - argilla: - image: argilla/argilla-server - ports: - - "6900:6900" - env: - ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200" - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry config installer.max-workers 10 - poetry install --no-interaction + uses: ./.github/workflows/python/run_notebooks.yml - - name: Configure Poetry for notebooks and run - env: - AA_TOKEN: ${{ secrets.AA_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - ARGILLA_API_URL: "http://localhost:6900/" - ARGILLA_API_KEY: "argilla.apikey" - CLIENT_URL: "https://api.aleph-alpha.com" - run: | - ./scripts/notebook_runner.sh + \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..5427c44dc --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,44 @@ +on: + workflow_call: +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + poetry config installer.max-workers 10 + poetry install --no-interaction + + - name: set PY for pre-commit + run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + + - uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Run linters + run: | + ./scripts/lint.sh diff --git a/.github/workflows/run_notebooks.yml b/.github/workflows/run_notebooks.yml new file mode 100644 index 000000000..f84bff9de --- /dev/null +++ b/.github/workflows/run_notebooks.yml @@ -0,0 +1,51 @@ +on: + workflow_call: +jobs: + run-notebooks: + runs-on: ubuntu-latest + services: + argilla-elastic-search: + image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 + env: + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + discovery.type: "single-node" + xpack.security.enabled: "false" + argilla: + image: argilla/argilla-server + ports: + - "6900:6900" + env: + ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + poetry config installer.max-workers 10 + poetry install --no-interaction + - name: Configure Poetry for notebooks and run + env: + AA_TOKEN: ${{ secrets.AA_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + ARGILLA_API_URL: "http://localhost:6900/" + ARGILLA_API_KEY: "argilla.apikey" + CLIENT_URL: "https://api.aleph-alpha.com" + run: | + ./scripts/notebook_runner.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..c92df63bf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,64 @@ +on: + workflow_call: +jobs: + test: + runs-on: ubuntu-latest + services: + argilla-elastic-search: + image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 + env: + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + discovery.type: "single-node" + xpack.security.enabled: "false" + argilla: + image: argilla/argilla-server + ports: + - "6900:6900" + env: + ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200" + open-telemetry-trace-service: + env: + COLLECTOR_OTLP_ENABLED: "true" + ports: + - "4317:4317" + - "4318:4318" + - "16686:16686" + image: jaegertracing/all-in-one:1.35 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + poetry config installer.max-workers 10 + poetry install --no-interaction + + - name: Run pytest + env: + AA_TOKEN: ${{ secrets.AA_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + ARGILLA_API_URL: "http://localhost:6900/" + ARGILLA_API_KEY: "argilla.apikey" + CLIENT_URL: "https://api.aleph-alpha.com" + run: | + ./scripts/test.sh