Merge branch 'main' into dependabot/npm_and_yarn/client/jsondiffpatch…

…-0.6.0
SURFscz · Apr 5, 2024 · 54057e1 · 54057e1
2 parents 6843409 + 0a4f99f
commit 54057e1
Show file tree

Hide file tree

Showing 233 changed files with 9,264 additions and 2,187 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -16,7 +16,8 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: surfscz/sram-sbs
+  IMAGE_NAME_SBS: surfscz/sram-sbs
+  IMAGE_NAME_SERVER: surfscz/sram-sbs-server
 
 jobs:
   Server_tests:
@@ -28,7 +29,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     services:
       # How to use MySQL
@@ -53,21 +54,32 @@ jobs:
           - 6379:6379
 
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Setup mysql server
         run: >
           mysql -uroot -proot -h127.0.0.1 -e "
             DROP DATABASE IF EXISTS sbs_test;
             CREATE DATABASE IF NOT EXISTS sbs_test DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
+            CREATE DATABASE IF NOT EXISTS sbs_gw0 DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
+            CREATE DATABASE IF NOT EXISTS sbs_gw1 DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
+            CREATE DATABASE IF NOT EXISTS sbs_gw2 DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
+            CREATE DATABASE IF NOT EXISTS sbs_gw3 DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
             CREATE USER 'sbs'@'%' IDENTIFIED BY 'sbs';
             GRANT ALL PRIVILEGES ON *.* TO 'sbs'@'%' WITH GRANT OPTION;
           "
+      - name: Load SBS schema
+        # not strictly necessary, but it avoids running 200+ migrations during tests
+        run: |
+          mysql -uroot -proot -h127.0.0.1 sbs_gw0 < misc/sbs-db.sql;
+          mysql -uroot -proot -h127.0.0.1 sbs_gw1 < misc/sbs-db.sql;
+          mysql -uroot -proot -h127.0.0.1 sbs_gw2 < misc/sbs-db.sql;
+          mysql -uroot -proot -h127.0.0.1 sbs_gw3 < misc/sbs-db.sql;
       - name: Install SAML2 dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y libxml2-dev libxmlsec1-dev libxml2
-      # Run Checkout code
-      - name: Checkout
-        uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
@@ -101,11 +113,13 @@ jobs:
       - name: Run tests with coverage
         run: |
           cd ./server
-          coverage run -m pytest test --cov-report xml --cov=server
+          coverage run -m pytest test --cov-report xml --cov=server --numprocesses=4
         timeout-minutes: 20
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
         if: success()
 
 
@@ -126,7 +140,7 @@ jobs:
         id: yarn-cache-dir-path
         run: echo "DIR=$(yarn cache dir)" >> $GITHUB_OUTPUT
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         id: yarn-cache
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.DIR }}
@@ -174,6 +188,7 @@ jobs:
       - name: Save version info
         run: |
           git log -1 > ./server/api/git.info
+          git describe --all --long > ./version.txt
 
       - name: Create build file
         run: |
@@ -225,7 +240,7 @@ jobs:
 
       - name: Create Release
         if: "github.ref_type=='tag'"
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           files: "${{steps.fetch_artifact.outputs.download-path}}/*"
 
@@ -238,7 +253,7 @@ jobs:
 
       - name: remove all previous "latest" releases
         if: "github.ref_type=='branch'"
-        uses: dev-drprasad/[email protected].2
+        uses: dev-drprasad/[email protected].3
         with:
           keep_latest: 0
           delete_tag_pattern: "branch+${{github.ref_name}}"
@@ -247,7 +262,7 @@ jobs:
 
       - name: Create Release for main/latest
         if: "github.ref_type=='branch'"
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           name: "Latest from branch ${{github.ref_name}}"
           tag_name: "branch+${{github.ref_name}}"
@@ -295,31 +310,116 @@ jobs:
         with:
           name: "sbs-build"
 
-      - name: Build Docker images
-        uses: docker/build-push-action@v5
-        with:
-          context: "."
-          pull: true
-          push: false
-
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
+      # The SBS image
+      - name: Extract metadata (tags, labels) for SBS image
+        id: meta-sbs
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_SBS }}
 
-      - name: Build and push Docker images
+      - name: Build and push Docker SBS image
         uses: docker/build-push-action@v5
         with:
           context: "."
-          pull: false
+          file: "Dockerfile.sbs"
+          pull: true
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-sbs.outputs.tags }}
+          labels: ${{ steps.meta-sbs.outputs.labels }}
+
+      # The SBS (apache) server image
+      - name: Extract metadata (tags, labels) for SBS-server image
+        id: meta-sbs-server
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_SERVER }}
+
+      - name: Build and push Docker SBS-server image
+        uses: docker/build-push-action@v5
+        with:
+          context: "."
+          file: "Dockerfile.sbs-server"
+          pull: true
+          push: true
+          tags: ${{ steps.meta-sbs-server.outputs.tags }}
+          labels: ${{ steps.meta-sbs-server.outputs.labels }}
+
+  Schema_pr:
+    name: Update schema cache
+
+    runs-on: ubuntu-latest
+
+    if: "github.ref == 'refs/heads/main'"
+
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup mysql server
+        run: >
+          mysql -uroot -proot -h127.0.0.1 -e "
+            DROP DATABASE IF EXISTS sbs;
+            CREATE DATABASE IF NOT EXISTS sbs DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
+            CREATE USER 'sbs'@'%' IDENTIFIED BY 'sbs';
+            GRANT ALL PRIVILEGES ON *.* TO 'sbs'@'%' WITH GRANT OPTION;
+          "
+      - name: Load SBS schema
+        # not strictly necessary, but it avoids running 200+ migrations during tests
+        run: |
+          mysql -uroot -proot -h127.0.0.1 sbs < misc/sbs-db.sql
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: 'pip'
+          cache-dependency-path: 'server/requirements/*.txt'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install pip setuptools wheel
+          pip install --upgrade pip
+          pip install -r ./server/requirements/test.txt
+
+      - name: Run alembic migrations
+        run: |
+          cd server
+          alembic --config migrations/alembic.ini upgrade head
+          echo alembic_current=$(alembic --config migrations/alembic.ini current) >> "$GITHUB_ENV"
+        env:
+          CONFIG: "config/test_config.yml"
+
+      - name: Dump latest database schema
+        run: |
+          echo "-- Dump of empty SBS database, alembic revision $alembic_current" > misc/sbs-db.sql
+          mysqldump -h 127.0.0.1 -u sbs --password=sbs --skip-comments sbs >> misc/sbs-db.sql
+
+      - name: Create Pull Request
+        uses: "peter-evans/create-pull-request@v6"
+        with:
+          add-paths: |
+            misc/sbs-db.sql
+          commit-message: "Update schema cache for Alembic revision ${{ env.alembic_current }}"
+          branch: "update-schema-cache"
+          title: "Update schema cache"
+          body: |
+            This PR updates the schema cache to the latest version.
+            It was created by a GitHub Actions workflow.
diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
@@ -0,0 +1,72 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '18 23 * * 2'
+  push:
+    branches: [ "main" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: false
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v2.2.4
+        with:
+          sarif_file: results.sarif
diff --git a/Dockerfile → Dockerfile.sbs b/Dockerfile → Dockerfile.sbs
@@ -1,4 +1,5 @@
-FROM python:3.11-slim-bookworm
+# First build SRAM SBS image
+FROM python:3.11-slim-bookworm AS sram-sbs
 
 # Do an initial clean up and general upgrade of the distribution
 ENV DEBIAN_FRONTEND noninteractive

diff --git a/Dockerfile.sbs-server b/Dockerfile.sbs-server
@@ -0,0 +1,10 @@
+# Build SRAM SBS (apache) server image
+FROM ghcr.io/openconext/openconext-basecontainers/apache2:latest AS sram-sbs-server
+RUN rm -f /etc/apache2/sites-enabled/*.conf
+RUN a2enmod proxy_wstunnel
+
+# Set the default workdir
+WORKDIR /opt
+
+#CMD ["bash"]
+
diff --git a/README.md b/README.md
@@ -44,13 +44,14 @@ Ensure MySQL is running and run the Python server with the correct local environ
 PROFILE=local ALLOW_MOCK_USER_API=1 CONFIG=config/test_config.yml python -m server
 ```
 
-With TESTING=1 no mails will be send. If you do want to validate the mails you can run a fake smtp server with:
+With TESTING=1 no mails will be sent. If you do want to validate the mails you can run a fake smtp server with:
 
 ```bash
 python -m smtpd -n -c DebuggingServer localhost:1025
 ```
 
-If you want the emails to be opened in the browser when developing add the `OPEN_MAIL_IN_BROWSER=1` to your environment
+If you want the emails to be opened in the browser when developing add the `OPEN_MAIL_IN_BROWSER=1` to your environment.
+Or even better, use https://mailpit.axllent.org/ and capture all emails send. 
 
 #### [Client](#client)
 
@@ -103,14 +104,18 @@ pytest test
 flake8 ./
 ```
 
-To generate coverage reports:
+To generate the coverage reports:
 
 ```bash
 source .venv/bin/activate
 cd server
 pytest --cov=server --cov-report html:htmlcov test
 open htmlcov/index.html
 ```
+Within PyCharm you must mark the `SBS/server/test` directory as Test sources root in order to execute `conftest.py`
+before tests are run. See https://intellij-support.jetbrains.com/hc/en-us/community/posts/12897247432338-PyCharm-unable-to-find-fixtures-in-conftest-py
+
+If you are getting errors in Pycharm when debugging, then have a look at https://youtrack.jetbrains.com/issue/PY-51495/PyCharm-debug-fails-upon-import-asyncio 
 
 To run all JavaScript tests: