Merge pull request #1 from alercebroker/feature/batch

Feature/batch

.github/workflows/integration_test.yml

@@ -16,25 +16,6 @@ jobs:
       matrix:
         python-version: [3.6, 3.7, 3.8]
 
-    # Service containers to run with `container-job`
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres
-        # Provide the password for postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-          # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -44,7 +25,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest numpy Cython
+          pip install pytest numpy Cython pytest-docker psycopg2
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Test with pytest
         run: |

docker-compose.yml

@@ -1,26 +1,27 @@
 version: "3"
 services:
-  features:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    restart: "always"
-    env_file:
-      - db_settings.env
-      - es_settings.env
+  feature:
+    build: .
     environment:
-      - CONSUMER_TOPICS=
-      - CONSUMER_SERVER=
-      - CONSUMER_GROUP_ID=
-      - PRODUCER_TOPIC=
-      - PRODUCER_SERVER=
-      - ES_PREFIX=
-      - ES_NETWORK_HOST=
-      - ES_NETWORK_PORT=
-      - DB_HOST=
-      - DB_USER=
-      - DB_PASSWORD=
-      - DB_PORT=
-      - DB_NAME=
-      - METRICS_HOST=
-      - METRICS_TOPIC=
+      - DB_ENGINE=postgres
+      - DB_HOST=${DB_HOST}
+      - DB_USER=${DB_USER}
+      - DB_PASSWORD=${DB_PASSWORD}
+      - DB_PORT=5432
+      - DB_NAME=${DB_NAME}
+      - CONSUMER_TOPICS=preprocess
+      - CONSUMER_SERVER=${CONSUMER_SERVER}
+      - CONSUMER_GROUP_ID=features_consumer_batch
+      - PRODUCER_TOPIC=features_batch
+      - PRODUCER_SERVER=${PRODUCER_SERVER}
+      - STEP_VERSION=features_1.0.1_dev
+      - FEATURE_VERSION=features_1.0.1_dev
+      - METRICS_HOST=${METRICS_HOST}
+      - METRICS_TOPIC=${METRICS_TOPIC}
+      - CONSUME_TIMEOUT=60
+      - CONSUME_MESSAGES=10
+    volumes:
+      - ./:/app
+    stdin_open: true
+    tty: true
+    command: /bin/bash