CI testing

.github/workflows/ci.yml

@@ -38,22 +38,22 @@ jobs:
         run: pipx run pre-commit run --all-files --show-diff-on-failure
       - name: Install tox-gh plugin
         run: python -m pip install tox-gh>=1.2
-      - name: Build package distribution files
-        run: tox -e clean,build
-      - name: Record the paths of wheel and source tarball distributions
-        id: distribution-paths
-        run: |
-          echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT
-          echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT
-      - name: Store the distribution files for use in other stages
-        # `tests`, `pypi-publish`, and `docker-publish` will use the same
-        # pre-built distributions, so we make sure to release the exact
-        # same package that was tested
-        uses: actions/upload-artifact@v3
-        with:
-          name: python-distribution-files
-          path: dist/
-          retention-days: 1
+      # - name: Build package distribution files
+      #   run: tox -e clean,build
+      # - name: Record the paths of wheel and source tarball distributions
+      #   id: distribution-paths
+      #   run: |
+      #     echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT
+      #     echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT
+      # - name: Store the distribution files for use in other stages
+      #   # `tests`, `pypi-publish`, and `docker-publish` will use the same
+      #   # pre-built distributions, so we make sure to release the exact
+      #   # same package that was tested
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: python-distribution-files
+      #     path: dist/
+      #     retention-days: 1
         # Prevent scheduled workflow from ever being paused
       - name: Keepalive Workflow
         uses: gautamkrishnar/[email protected]
@@ -86,6 +86,8 @@ jobs:
         run: python -m pip install tox-gh>=1.2
       - name: Setup test suite
         run: tox -vv --notest
+      - name: Install dependencies
+        run: ./dev_setup.sh {{{ matrix.python }}}
       - name: Run tests
         env:
           # New variables should match a `passenv` pattern under `[testenv]` in tox.ini

.gitignore

@@ -60,3 +60,7 @@ MANIFEST
 
 # Metaflow files
 .metaflow
+
+# Virtual Environment Files
+/venv
+/py-orca-venv-*

CONTRIBUTING.md

@@ -101,19 +101,18 @@ This often provides additional considerations and avoids unnecessary work.
    cd orca
    ```
 
-4. Install `pipx` to easily run Python CLI tools like `tox` and `pipenv`.
-
 4. Create an isolated virtual environment containing package dependencies,
    including those needed for development (*e.g.* testing, documentation) by running:
 
    ```console
-   pipx run tox -e pipenv
+   ./dev_setup.sh <your_python_version>
+   source py-orca-venv-<your_python_version>/bin/activate
    ```
 
 5. Install [pre-commit] hooks:
 
    ```
-   pipenv run pre-commit install
+   pre-commit install
    ```
 
    `orca` comes with a lot of hooks configured to automatically help the

README.md

@@ -24,21 +24,6 @@ Refer to [`.env.example`](.env.example) for the format of their values as well a
 
 Once your environment is set, you can create a virtual environment, install the Python dependencies, and run the demonstration script (after downloading it) as follows. Note that you will need to update the `s3_prefix` parameter so that it points to an S3 bucket that is accessible to your Tower workspace.
 
-### Using pipenv
-```bash
-# Create and activate a Python virtual environment (tested with Python 3.11)
-pipenv shell
-
-# Install Python dependencies
-pipenv install
-
-# If development dependencies are required to be installed
-pipenv install -d
-
-# Run the script using an example dataset
-python3 demo.py run --dataset_id 'syn51514585' --s3_prefix 's3://orca-service-test-project-tower-bucket/outputs'
-```
-
 ### Manually creating a virtual environment
 ```bash
 # Create and activate a Python virtual environment (tested with Python 3.10)

dev_setup.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Usage: ./dev_setup.sh <python_version>
+
+# Check if Python version argument is provided
+if [ -z "$1" ]; then
+  echo "Please provide the Python version as an argument."
+  exit 1
+fi
+# Check if Python version is supported
+if [ "$1" != "3.10" ] && [ "$1" != "3.11" ]; then
+  echo "Unsupported Python version. Please use 3.10 or 3.11."
+  exit 1
+fi
+# Set up and activate a Python 3.11 virtual environment
+python$1 -m venv py-orca-venv-$1
+source py-orca-venv-$1/bin/activate
+# Upgrade pip to latest version
+pip install --upgrade pip
+# Install airflow with constraints
+pip install -r requirements-airflow.txt
+# Install py-orca
+pip install -e '.[all,testing,dev]'

requirements-airflow.txt

@@ -0,0 +1,2 @@
+-c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.2/constraints-3.10.txt
+apache-airflow==2.7.2

tox.ini

@@ -94,14 +94,13 @@ commands =
     python -m twine upload {posargs:--repository {env:TWINE_REPOSITORY:testpypi}} dist/*
 
 
-[testenv:pipenv]
-description =
-    Refresh the Pipfile.lock file based on the latest dependencies in setup.cfg
-skip_install = True
-changedir = {toxinidir}
-setenv =
-    PIPENV_IGNORE_VIRTUALENVS = 1
-deps = pipenv
-commands =
-    pipenv lock --dev
-    pipenv install --dev
+; [testenv:pipenv]
+; description =
+;     Set up the python virtual environment and install dependencies using pip
+; skip_install = True
+; changedir = {toxinidir}
+; setenv =
+;     PIPENV_IGNORE_VIRTUALENVS = 1
+; deps = pip
+; commands =
+;     ./dev_setup.sh