diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 8bdddb7f2..6d9f14617 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -9,206 +9,14 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true - jobs: doctest: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry config installer.max-workers 10 - poetry install --no-interaction - - - name: install Sphinx - run: sudo apt-get update -y && sudo apt-get install python3-sphinx - - - name: run doctest - env: - AA_TOKEN: ${{ secrets.AA_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - CLIENT_URL: "https://api.aleph-alpha.com" - run: ./scripts/doctest.sh - + uses: ./.github/workflows/python/doctest.yml lint: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry config installer.max-workers 10 - poetry install --no-interaction - - - name: set PY for pre-commit - run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - - uses: actions/cache@v4 - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - - name: Run linters - run: | - ./scripts/lint.sh - + uses: ./.github/workflows/python/lint.yml test: - runs-on: ubuntu-latest - - services: - argilla-elastic-search: - image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 - env: - ES_JAVA_OPTS: "-Xms512m -Xmx512m" - discovery.type: "single-node" - xpack.security.enabled: "false" - argilla: - image: argilla/argilla-server - ports: - - "6900:6900" - env: - ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200" - open-telemetry-trace-service: - env: - COLLECTOR_OTLP_ENABLED: "true" - ports: - - "4317:4317" - - "4318:4318" - - "16686:16686" - image: jaegertracing/all-in-one:1.35 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry config installer.max-workers 10 - poetry install --no-interaction - - - name: Run pytest - env: - AA_TOKEN: ${{ secrets.AA_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - ARGILLA_API_URL: "http://localhost:6900/" - ARGILLA_API_KEY: "argilla.apikey" - CLIENT_URL: "https://api.aleph-alpha.com" - run: | - ./scripts/test.sh - + uses: ./.github/workflows/python/test.yml run-notebooks: - runs-on: ubuntu-latest - - services: - argilla-elastic-search: - image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 - env: - ES_JAVA_OPTS: "-Xms512m -Xmx512m" - discovery.type: "single-node" - xpack.security.enabled: "false" - argilla: - image: argilla/argilla-server - ports: - - "6900:6900" - env: - ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200" - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry config installer.max-workers 10 - poetry install --no-interaction + uses: ./.github/workflows/python/run_notebooks.yml - - name: Configure Poetry for notebooks and run - env: - AA_TOKEN: ${{ secrets.AA_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - ARGILLA_API_URL: "http://localhost:6900/" - ARGILLA_API_KEY: "argilla.apikey" - CLIENT_URL: "https://api.aleph-alpha.com" - run: | - ./scripts/notebook_runner.sh + \ No newline at end of file diff --git a/.github/workflows/python/doctest.yml b/.github/workflows/python/doctest.yml new file mode 100644 index 000000000..b27282fc3 --- /dev/null +++ b/.github/workflows/python/doctest.yml @@ -0,0 +1,41 @@ +jobs: + doctest: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + poetry config installer.max-workers 10 + poetry install --no-interaction + + - name: install Sphinx + run: sudo apt-get update -y && sudo apt-get install python3-sphinx + + - name: run doctest + env: + AA_TOKEN: ${{ secrets.AA_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + CLIENT_URL: "https://api.aleph-alpha.com" + run: ./scripts/doctest.sh \ No newline at end of file diff --git a/.github/workflows/python/lint.yml b/.github/workflows/python/lint.yml new file mode 100644 index 000000000..ea9e9b55d --- /dev/null +++ b/.github/workflows/python/lint.yml @@ -0,0 +1,43 @@ +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + poetry config installer.max-workers 10 + poetry install --no-interaction + + - name: set PY for pre-commit + run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + + - uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Run linters + run: | + ./scripts/lint.sh \ No newline at end of file diff --git a/.github/workflows/python/run_notebooks.yml b/.github/workflows/python/run_notebooks.yml new file mode 100644 index 000000000..f00f14dab --- /dev/null +++ b/.github/workflows/python/run_notebooks.yml @@ -0,0 +1,56 @@ +jobs: + run-notebooks: + uses: ./.github/workflows/python/run_notebooks.yml + runs-on: ubuntu-latest + + services: + argilla-elastic-search: + image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 + env: + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + discovery.type: "single-node" + xpack.security.enabled: "false" + argilla: + image: argilla/argilla-server + ports: + - "6900:6900" + env: + ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + poetry config installer.max-workers 10 + poetry install --no-interaction + + - name: Configure Poetry for notebooks and run + env: + AA_TOKEN: ${{ secrets.AA_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + ARGILLA_API_URL: "http://localhost:6900/" + ARGILLA_API_KEY: "argilla.apikey" + CLIENT_URL: "https://api.aleph-alpha.com" + run: | + ./scripts/notebook_runner.sh diff --git a/.github/workflows/python/test.yml b/.github/workflows/python/test.yml new file mode 100644 index 000000000..257bda08e --- /dev/null +++ b/.github/workflows/python/test.yml @@ -0,0 +1,62 @@ +jobs: + test: + runs-on: ubuntu-latest + services: + argilla-elastic-search: + image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 + env: + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + discovery.type: "single-node" + xpack.security.enabled: "false" + argilla: + image: argilla/argilla-server + ports: + - "6900:6900" + env: + ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200" + open-telemetry-trace-service: + env: + COLLECTOR_OTLP_ENABLED: "true" + ports: + - "4317:4317" + - "4318:4318" + - "16686:16686" + image: jaegertracing/all-in-one:1.35 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + poetry config installer.max-workers 10 + poetry install --no-interaction + + - name: Run pytest + env: + AA_TOKEN: ${{ secrets.AA_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + ARGILLA_API_URL: "http://localhost:6900/" + ARGILLA_API_KEY: "argilla.apikey" + CLIENT_URL: "https://api.aleph-alpha.com" + run: | + ./scripts/test.sh \ No newline at end of file