setup airflow e2e execution in workflow (#1466)

neptune-ai · Sep 13, 2023 · eef5dd4 · eef5dd4
1 parent c210b40
commit eef5dd4
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 3 deletions.
diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml
@@ -637,3 +637,55 @@ jobs:
         with:
           slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }}
           job-status: ${{ needs.mlflow.result }}
+
+  airflow:
+    env:
+      NEPTUNE_PROJECT: e2e-tests/integrations
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.8", "3.10" ]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+    runs-on: ${{ matrix.os }}
+    name: 'airflow (${{ matrix.os }} - py${{ matrix.python-version }})'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.client_payload.pull_request.head.ref }}
+
+      - name: Install package
+        uses: ./.github/actions/install-package
+        with:
+          python-version: ${{ matrix.python-version }}
+          os: ${{ matrix.os }}
+
+      - name: Install airflow
+        uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main
+        with:
+          python-version: ${{ matrix.python-version }}
+          working_directory: neptune-airflow
+
+      - name: Test integration
+        uses: neptune-ai/neptune-airflow/.github/actions/e2e@main
+        with:
+          working_directory: neptune-airflow
+
+  airflow-notify:
+    needs: [ airflow ]
+    runs-on: ubuntu-latest
+    if: (success() || failure()) && github.ref == 'refs/heads/master'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.client_payload.pull_request.head.ref }}
+
+      - name: Notify
+        uses: ./.github/actions/job-notify
+        with:
+          slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }}
+          job-status: ${{ needs.airflow.result }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,7 @@
-## [UNRELEASED] neptune 1.6.4
+## [UNRELEASED] neptune 1.7.0
+
+### Features
+- Added support for `airflow` integration ([#1466](https://github.com/neptune-ai/neptune-client/pull/1466))
 
 ### Changes
 - Add handling of project limits [#1456](https://github.com/neptune-ai/neptune-client/pull/1456)

diff --git a/pyproject.toml b/pyproject.toml
@@ -84,7 +84,8 @@ neptune-xgboost = { version = "*", optional = true }
 transformers = { version = "*", optional = true }
 zenml = { version = "*", optional = true, python = "<3.9" }
 neptune-aws = { version = "*", optional = true }
-mlflow = { version = ">1.30", optional = true }
+neptune-mlflow = { version = "*", optional = true }
+neptune-airflow = { version = "*", optional = true }
 
 [tool.poetry.extras]
 aws = ["neptune-aws"]
@@ -103,7 +104,8 @@ tensorboard = ["neptune-tensorboard"]
 transformers = ["transformers"]
 xgboost = ["neptune-xgboost"]
 zenml = ["zenml"]
-mlflow = ["mlflow"]
+mlflow = ["neptune-mlflow"]
+airflow = ["neptune-airflow"]
 dev = [
     "altair",
     "bokeh",