Long overdue refactoring for quality of life and performance improvem…

…ents. (#140)

This PR represents a near-total refactoring of the existing Chassis code to fix bugs and add some long-awaited features. Even though this is a significant refactor, there should be no* breaking changes for users of the Chassis SDK.

Notable improvements:
1. Chassis now supports local Docker builds. Using a remote build server is still supported but is no longer required.
2. The dependency on MLFlow has been removed. Any function that can be cloudpickle'd is now supported (MLFlow also uses cloudpickle under the hood.)
3. Models now support multiple inputs and outputs per inference.
4. The number of PIP dependencies to install the Chassis SDK has been dramatically reduced. For some features (like kserve support), there are optional dependencies that can be used to bring in those extra dependencies.
5. The use of conda to satisfy dependencies is no longer required. This, combined with the removal of MLFlow results is an up to 10x size reduction in the generated model container.
6. Due to all of the above, container build times are now 2-5x faster.

In addition, the remote build server has been completely rewritten (in Rust). It no longer has any dependencies on object storage and has much better support for parallel builds. In addition, kaniko has been replaced by BuildKit which enables multi-platform images to be built. Finally, the Helm chart has been updated to allow changing all Kubernetes values like the amount of resources to allocate to the BuildKit pods, various timeouts, etc.

.editorconfig

@@ -0,0 +1,16 @@
+# Editor configuration, see http://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 4
+insert_final_newline = true
+trim_trailing_whitespace = true
+# max_line_length = 100
+
+[*.yaml]
+indent_size = 2
+
+[*.yml]
+indent_size = 2

.github/workflows/CI_PR.yml

@@ -1,96 +1,101 @@
-# This is a basic workflow to help you get started with Actions
-
 name: CI_PR
 
 # Controls when the action will run.
 on:
-  # Triggers the workflow on push or pull request events but only for the main branch
-  # push:
-    # branches: [ main ]
-    # tags: v*
-
-  pull_request: 
-
-  # Allows you to run this workflow manually from the Actions tab
+  pull_request:
+    branches:
+      - main
+      - release-v*
+
+      # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
+
   lint:
+    name: Run Python lint and type checks
     runs-on: ubuntu-latest
-
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - name: Setup Repo
-        uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          submodules: recursive 
+          # Semantic version range syntax or exact version of a Python version
+          python-version: 3.9
+          cache: pip
+          cache-dependency-path: |
+            packages/chassisml/pyproject.toml
 
-      - name: Setup Environment Variable
+      - name: Install dependencies
         run: |
-          export CHASSIS_URL=${{secrets.CHASSIS_URL}}
-          export DOCKER_USER=${{secrets.DOCKER_USER}}
-          export DOCKER_PASS=${{secrets.DOCKER_PASS}}
-          export MODZY_BASE_URL=${{secrets.MODZY_BASE_URL}}
-          export MODZY_API_KEY=${{secrets.MODZY_API_KEY}}
-          
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+          python -m pip install --upgrade pip
+          pip install ./packages/chassisml[test]
+
+      - name: Lint
+        run: tox -e lint
+
+  #      - name: Verify Types
+  #        run: tox -e type
+
+  tests:
+    name: Run tests for ${{ matrix.os }}/${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        include:
+          - python-version: "3.8"
+            tox-name: py38
+          - python-version: "3.9"
+            tox-name: py39
+          - python-version: "3.10"
+            tox-name: py310
+          - python-version: "3.11"
+            tox-name: py311
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          # Semantic version range syntax or exact version of a Python version
-          python-version: 3.8
+          python-version: "${{ matrix.python-version }}"
+          cache: pip
+          cache-dependency-path: |
+            packages/chassisml/pyproject.toml
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Lint with flake8
+          pip install ./packages/chassisml[test]
+
+      - name: Run tests
         run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --extend-ignore E303,E501,E712,E111,F821,E225 --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-  tests:
+          tox -e ${{ matrix.tox-name }}
+
+  pip:
+    name: Build SDK
+    needs: [lint, tests]
     runs-on: ubuntu-latest
-    if: true
-    env:
-      CHASSIS_URL: ${{secrets.CHASSIS_URL}}
-      DOCKER_USER: ${{secrets.DOCKER_USER}}
-      DOCKER_PASS: ${{secrets.DOCKER_PASS}}
-      MODZY_BASE_URL: ${{secrets.MODZY_BASE_URL}}
-      MODZY_API_KEY: ${{secrets.MODZY_API_KEY}}
-
-    steps:   
-    # new
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive 
+    steps:
+      - uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          # Semantic version range syntax or exact version of a Python version
-          python-version: '3.8'
+          python-version: 3.9
+          cache: pip
+          cache-dependency-path: |
+            packages/chassisml/pyproject.toml
 
-      - name: Upgrade pip
+      - name: Install pypa/build
         run: |
-          # install pip=>20.1 to use "pip cache dir"
-          python3 -m pip install --upgrade pip
-      - name: Get pip cache dir
-        id: pip-cache
-        run: echo "::set-output name=dir::$(pip cache dir)"
-
-      - name: Cache dependencies
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-        
-      - name: Install dependencies
-        run: python3 -m pip install -r ./tests/requirements.txt                      
+          pip install build
 
-      - name: Test with Chassis specific testing scheme
+      - name: Build distribution
         run: |
-          python -m tests.test --ci True
-  
+          python3 -m build --sdist --wheel --outdir dist/ packages/chassisml