Merge pull request #2 from earthcube/dev_containerize

Add Containerize workflow

.github/workflows/contanerize.yaml

@@ -1,19 +1,55 @@
 name: Dockerize  Dagster
 on:
-  publish:
+  push:
+    branches:
+      - "**"
+    tags:
+      - "v*.*.*"
 
 # https://github.com/marketplace/actions/publish-docker
 # https://github.com/docker/build-push-action
 defaults:
   run:
-    working-directory: .
+    working-directory:  dagster/implnets
 jobs:
   deploy:
-    name: Dockerize Flask
+    name: Dockerize  Scheduler for Project
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        project: [ "eco", "iow", "oih" ]
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version:  '3.10'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+# long version with lint,
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install flake8 pytest
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+#          pip install build
+#      - name: Lint with flake8
+#        run: |
+#          # stop the build if there are Python syntax errors or undefined names
+#          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+#          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+#          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+#      - name: Build package
+#        run: python -m build
+#      - name: Build tooling
+#        run: |
+#          cd ./tooling/cfgBuilder/${{ matrix.project }}
+#          python cfgBuilder.py -s https://foo.us/sitemap.xml
+      - name: Generate
+        run: |
+          python pygen.py -cf ./configs/${{ matrix.project }}/gleanerconfig.yaml -od ./generatedCode/implnet-${{ matrix.project }}/output  -td ./templates/v1   -d 7
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
@@ -27,7 +63,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: gleanerio/dagster
+          images: nsfearthcube/dagster-${{ matrix.project }}
           flavor: |
             latest=true
           tags: |
@@ -36,11 +72,13 @@ jobs:
             type=semver,pattern={{version}}
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           push: true
+          build-args:
+            implnet=${{ matrix.project }}
           file: ./build/Dockerfile
-          context: .
+          context:  "{{defaultContext}}:dagster/implnets"
           tags: ${{ steps.meta.outputs.tags }}
 #          tags: nsfearthcube/ec_facets_client:latest
           labels: ${{ steps.meta.outputs.labels }}

dagster/implnets/VERSION

@@ -1 +1 @@
-0.0.65
+0.0.67

dagster/implnets/deployment/compose_project.yaml

@@ -20,7 +20,7 @@ networks:
     name:   traefik_proxy
 services:
     dagster-dagit:
-        image: docker.io/fils/dagster_${PROJECT}:0.0.64
+        image: docker.io/nsfearthcube/dagster-${PROJECT}:latest
         secrets:
           - MINIO_ROOT_ACCESS_KEY
           - MINIO_ROOT_SECRET_KEY
@@ -82,7 +82,7 @@ services:
               - "traefik.http.middlewares.sched-${PROJECT}.headers.accesscontrolmaxage=100"
               - "traefik.http.middlewares.sched-${PROJECT}.headers.addvaryheader=true"
     dagster-daemon:
-        image: docker.io/fils/dagster_${PROJECT}:0.0.64
+        image: docker.io/nsfearthcube/dagster-${PROJECT}:latest
         secrets:
           - MINIO_ROOT_ACCESS_KEY
           - MINIO_ROOT_SECRET_KEY

dagster/implnets/pyproject.toml

@@ -1,3 +1,9 @@
+[project]
+name = "GleanerIO Scheduler"
+version = "0.0.67"
+description = "Tools for harvesting and loading on a schedule from gleaner configurations"
+# readme =  "README.md"
+
 [tool.black]
 line-length = 99
 target-version = ['py37']

dagster/implnets/requirements.txt

@@ -1,6 +1,6 @@
-dagit==0.14.3
-dagster-postgres==0.14.3
-dagster==0.14.3
+dagit>=1.3.8
+dagster-postgres==0.19.8
+dagster>=1.3.8
 ipython-genutils==0.2.0
 advertools==0.13.2
 minio==7.1.13