From a5a83b1c234695eafe6e3aa867cad8e1e0d741a0 Mon Sep 17 00:00:00 2001 From: wojciechwoszczek Date: Wed, 6 Dec 2023 18:31:50 +0100 Subject: [PATCH] v1 --- .github/workflows/backend.yml | 98 ++++++++++++++++++++++++++++++++++ .github/workflows/frontend.yml | 58 ++++++++++++++++++++ .github/workflows/main.yml | 21 -------- requirements.txt | 19 ++++--- 4 files changed, 169 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/backend.yml create mode 100644 .github/workflows/frontend.yml delete mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml new file mode 100644 index 0000000..ccfdbbe --- /dev/null +++ b/.github/workflows/backend.yml @@ -0,0 +1,98 @@ + name: Backend model CI/CD + + run-name: Action executed by ${{ github.actor }} - Backend CI/CD + + on: + push: + branches: + - main + - github-actions + pull_request: + branches: + - main + + jobs: + ci_build_and_test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Environment setup # Set up with a specific version of Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + cache: pip + + - name: Cache # Cache dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: ${{ runner.os }}-pip- + + - name: Create .env file + run: | + echo "MLFLOW_TRACKING_URI=${{ secrets.MLFLOW_TRACKING_URI }}" >> .env + echo "MLFLOW_TRACKING_USERNAME=${{ secrets.MLFLOW_TRACKING_USERNAME }}" >> .env + echo "MLFLOW_TRACKING_PASSWORD=${{ secrets.MLFLOW_TRACKING_PASSWORD }}" >> .env + + - name: Install packages # Install dependencies + run: pip install pytest mlflow python-dotenv + + - name: Run training tests + run: | + pytest tests/test_mlflow.py + pytest tests/test_train_model.py + continue-on-error: true + + + cd_push_to_dockerhub: + needs: ci_build_and_test + runs-on: ubuntu-latest + if: success() + + steps: + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Environment setup # Set up with a specific version of Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + cache: pip + + - name: Cache # Cache dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: ${{ runner.os }}-pip- + + - name: Install packages # Install dependencies + run: pip install dvc docker + + - name: Pull data + run: | + dvc remote modify origin --local auth basic + dvc remote modify origin --local user ${{ secrets.DAGSHUB_USERNAME }} + dvc remote modify origin --local password ${{ secrets.DAGSHUB_TOKEN }} + dvc pull -r origin models/trained_model.pt + + - name: Docker login + run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASSWORD }} + + - name: Build + run: docker build -f Dockerfile-backend.txt . -t app-beans-backend + + - name: Tags + run: | + docker tag app-beans-backend ${{ secrets.DOCKER_USER }}/app-beans-backend:${{ github.sha }} + docker tag app-beans-backend ${{ secrets.DOCKER_USER }}/app-beans-backend:latest + + - name: Push + run: | + docker push ${{ secrets.DOCKER_USER }}/app-beans-backend:${{ github.sha }} + docker push ${{ secrets.DOCKER_USER }}/app-beans-backend:latest \ No newline at end of file diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 0000000..75355e0 --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,58 @@ + name: Frontend CI/CD + + run-name: Action executed by ${{ github.actor }} - Frontend CI/CD + + on: + push: + paths: + - 'src/web/**' + branches: + - main + - github-actions + pull_request: + paths: + - 'src/web/**' + branches: + - main + + jobs: + + cd_push_to_dockerhub: + runs-on: ubuntu-latest + + steps: + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Environment setup # Set up with a specific version of Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + cache: pip + + - name: Cache # Cache dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: ${{ runner.os }}-pip- + + - name: Install packages # Install dependencies + run: pip install docker + + - name: Docker login + run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASSWORD }} + + - name: Build + run: docker build -f Dockerfile-frontend.txt . -t app-beans-frontend + + - name: Tags + run: | + docker tag app-beans-frontend ${{ secrets.DOCKER_USER }}/app-beans-frontend:${{ github.sha }} + docker tag app-beans-frontend ${{ secrets.DOCKER_USER }}/app-beans-frontend:latest + + - name: Push + run: | + docker push ${{ secrets.DOCKER_USER }}/app-beans-frontend:${{ github.sha }} + docker push ${{ secrets.DOCKER_USER }}/app-beans-frontend:latest \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 1cbf192..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -# name: Backend - -# on: -# push: -# branches: -# - master -# pull_request: -# branches: -# - master - -# jobs: -# build-and-test: -# runs-on: ubuntu-22.04 - -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 -# with: -# sparse-checkout: | -# .github -# src diff --git a/requirements.txt b/requirements.txt index e66288f..aa91466 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,22 @@ flake8==6.0.0 -numpy==1.24.2 +numpy~=1.23.5 scikit-learn==1.3.0 codecarbon==2.3.1 -pynblint==0.1.5 +#pynblint==0.1.5 great_expectations==0.17.22 Pillow==10.0 pytest==7.4.3 -torchvision==0.16.1 +torchvision~=0.16.0 python-multipart==0.0.6 -fastapi==0.68.0 +fastapi~=0.104.1 uvicorn[standard]==0.24.0.post1 httpx==0.21.0 -datasets==2.15.0 -Flask==3.0.0 \ No newline at end of file +datasets~=2.14.5 +dvc>=3.0.0 +docker==6.1.3 +torch~=2.1.0 +pydantic~=1.10.13 +flask~=3.0.0 +python-dotenv~=1.0.0 +pandas~=2.0.3 +mlflow~=2.8.0 \ No newline at end of file