EFRS-1114: Google Coral Support for FaceNet - Updated (1.2.0)

.dockerignore

@@ -1,2 +1 @@
-.git
-/embedding-calculator/sample_images
+.git

.env

@@ -10,9 +10,17 @@ email_from=
 email_password=
 enable_email_server=false
 save_images_to_db=true
-compreface_api_java_options=-Xmx8g
-compreface_admin_java_options=-Xmx8g
-ADMIN_VERSION=0.5.1
-API_VERSION=0.5.1
-FE_VERSION=0.5.1
-CORE_VERSION=0.5.1
+compreface_api_java_options=-Xmx4g
+compreface_admin_java_options=-Xmx1g
+max_file_size=5MB
+max_request_size=10M
+max_detect_size=640
+uwsgi_processes=2
+uwsgi_threads=1
+connection_timeout=10000
+read_timeout=60000
+ADMIN_VERSION=1.2.0
+API_VERSION=1.2.0
+FE_VERSION=1.2.0
+CORE_VERSION=1.2.0
+POSTGRES_VERSION=1.2.0

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,25 +8,40 @@ assignees: ''
 ---
 
 **Describe the bug**
+
 A clear and concise description of what the bug is.
 
 **To Reproduce**
+
 Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
 **Expected behavior**
+
 A clear and concise description of what you expected to happen.
 
 **Screenshots**
+
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
+
  - OS: [e.g. iOS]
  - Browser [e.g. chrome, safari]
  - Version [e.g. 22]
 
+**Logs**
+
+Run those commands and attach result to the ticket:
+
+`docker ps`
+
+`docker-compose logs`
+
+
 **Additional context**
+
 Add any other context about the problem here.

.github/workflows/AWS-CompreFace-image.yml

@@ -0,0 +1,59 @@
+name: (AWS) Create CompreFace image for new release
+
+on: 
+  workflow_dispatch:
+    inputs:
+      release:
+        description: release zip (e.g., https://github.com/exadel-inc/CompreFace/releases/download/v1.0.0/CompreFace_1.0.0.zip)
+        required: true
+      version:
+        description: version (e.g., 1.0.0)
+        required: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_KEY_ACCESS }}
+        aws-region: us-east-1
+    - name: Create Security Group
+      run: |
+        export SECURITY_GROUP_ID=$(aws ec2 create-security-group --group-name MySecurityGroup --description "My security group" --query 'GroupId' --output text)
+        aws ec2 wait security-group-exists --group-ids ${SECURITY_GROUP_ID}
+        aws ec2 authorize-security-group-ingress --group-id ${SECURITY_GROUP_ID} --protocol tcp --port 22 --cidr 0.0.0.0/0
+        echo "SECURITY_GROUP_ID=$SECURITY_GROUP_ID" >> $GITHUB_ENV
+    - name: Run Instance
+      run: |
+        echo ${SECURITY_GROUP_ID}
+        export INSTANCE_ID=$(aws ec2 run-instances --image-id ami-04e612d1108883950 --count 1 --instance-type t2.medium --key-name IharB --security-group-ids ${SECURITY_GROUP_ID} --subnet-id subnet-080dc6a6ed9580c77 --query 'Instances[0].InstanceId' --output text)
+        aws ec2 wait instance-running --instance-ids ${INSTANCE_ID}
+        echo "INSTANCE_ID=$INSTANCE_ID" >> $GITHUB_ENV
+        sleep 10
+    - name: Install Release
+      env:
+         RELEASE: ${{ github.event.inputs.release }}
+         SSH_KEY: ${{secrets.SSH_KEY}}
+      run: |
+        echo "$SSH_KEY" > private_key && chmod 600 private_key
+        export INSTANCE_IP_ADDRESS_EXTERNAL=$(aws ec2 describe-instances --instance-id ${INSTANCE_ID} --query 'Reservations[].Instances[].NetworkInterfaces[].Association.PublicIp' --output text)
+        echo $INSTANCE_IP_ADDRESS_EXTERNAL
+        ssh -i private_key -oStrictHostKeyChecking=no ec2-user@$INSTANCE_IP_ADDRESS_EXTERNAL "wget -q -O tmp.zip '$RELEASE' && unzip -o tmp.zip && rm tmp.zip && docker-compose stop && docker-compose rm --force && docker image prune -a --force && docker-compose up -d && rm /home/ec2-user/.ssh/authorized_keys && sudo rm /root/.ssh/authorized_keys"
+    - name: Stop Instance
+      run: |
+        aws ec2 stop-instances --instance-ids $INSTANCE_ID
+        aws ec2 wait instance-stopped --instance-ids $INSTANCE_ID
+    - name: Create Image
+      env:
+         VERSION: ${{ github.event.inputs.version }}
+      run: |
+        export IMAGE_ID=$(aws ec2 create-image --instance-id $INSTANCE_ID --name "CompreFace_${VERSION}" --description "CompreFace Base Image" --query 'ImageId' --output text)
+        echo  "CompreFace Base Image id of version ${VERSION} : ${IMAGE_ID}"
+        aws ec2 wait image-available --image-ids ${IMAGE_ID}
+    - name: Delete resources
+      run: |
+        aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+        aws ec2 wait instance-terminated --instance-ids $INSTANCE_ID
+        aws ec2 delete-security-group --group-name MySecurityGroup

.github/workflows/AWS-Сompreface-packer-image.yml

@@ -0,0 +1,40 @@
+---
+name: (AWS) Сompreface packer image
+on: 
+  workflow_dispatch:
+jobs:
+  packer:
+    runs-on: ubuntu-latest
+    name: packer
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_KEY_ACCESS }}
+          aws-region: us-east-1
+
+      - name: Initialize Packer Template
+        uses: hashicorp/packer-github-actions@master
+        with:
+          command: init
+
+      - name: Validate Template
+        uses: hashicorp/packer-github-actions@master
+        with:
+          command: validate
+          arguments: -syntax-only
+          target: aws-compreface.pkr.hcl
+
+      - name: Build Artifact
+        uses: hashicorp/packer-github-actions@master
+        with:
+          command: build
+          arguments: "-color=true -on-error=abort"
+          target: aws-compreface.pkr.hcl
+        env:
+          PACKER_LOG: 1

.github/workflows/Azure-Compreface-AIB-image.yml

@@ -0,0 +1,51 @@
+---
+name: (Azure) Сompreface AIB image
+on: 
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version (e.g., 1.0.0)
+        required: true
+env:
+  RESOURCE_GROUP_NAME: compreFaceGallery-RG
+  MANAGED_IDENTITY: compreFace-MI
+  GALLERY_NAME: compreFaceGallery
+  IMAGE_NAME: compreFace
+  VERSION: ${{ github.event.inputs.version }}
+jobs:
+  build-image:
+    runs-on: ubuntu-latest    
+    steps:
+
+    - name: Login via Az module
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZURE_CREDENTIALS}}
+
+    - name: Build custom VM image
+      id: imageBuilder      
+      uses: azure/build-vm-image@v0
+      with:        
+        resource-group-name: "${{ env.RESOURCE_GROUP_NAME }}"
+        managed-identity: "${{ env.MANAGED_IDENTITY }}"
+        location: 'eastus'
+        source-os-type: 'linux'
+        source-image-type: 'PlatformImage'     
+        source-image: Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:latest
+        vm-size: 'Standard_B2s'
+        dist-type: 'SharedImageGallery'
+        dist-resource-id: "/subscriptions/${{secrets.AZURE_SUBSCRIPTION_ID}}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/providers/Microsoft.Compute/galleries/${{ env.GALLERY_NAME }}/images/${{ env.IMAGE_NAME }}/versions/${{ env.VERSION }}"
+        dist-location: 'eastus'
+        customizer-script: |
+          apt update
+          apt upgrade -y
+          apt install -y docker.io unzip
+          docker version
+          curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
+          chmod +x /usr/local/bin/docker-compose
+          docker-compose version
+          chmod 666 /var/run/docker.sock
+          wget -q -O tmp.zip "https://github.com/exadel-inc/CompreFace/releases/download/v${{ env.VERSION }}/CompreFace_${{ env.VERSION }}.zip" && unzip tmp.zip && rm tmp.zip
+          sed -i "s|8000:|80:|g" docker-compose.yml
+          docker-compose pull --quiet
+          docker-compose up -d

.github/workflows/Build-Deploy-auto.yml

@@ -0,0 +1,115 @@
+name: Build and Deploy CompreFace on push
+
+on: 
+  push:
+    branches:
+      - master
+      - '1.2.x'
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    # Map steps outputs to a job outputs.
+    # We need to share it between build and deploy jobs.
+    outputs:
+      registry_path: ${{ steps.registry_path.outputs.registry_path }}
+      tag: ${{ steps.tag_vars.outputs.tag }}
+      tag_latest: ${{ steps.tag_vars.outputs.tag_latest }}
+      env_name: ${{ steps.env_var.outputs.env_name }}
+
+    steps:
+    - name: Checkout Repo 
+      uses: actions/checkout@v3
+
+    - name: Set registry path output
+      id: registry_path
+      run: echo "registry_path=${{ env.REGISTRY }}/exadel-inc/compreface/" >> $GITHUB_OUTPUT
+
+    - name: Set tags from git output
+      id: tag_vars
+      run: |
+        echo "tag=${{ github.ref_name }}-$(git rev-parse HEAD | cut -c 1-7 | tr -d '\n')" >> $GITHUB_OUTPUT
+        echo "tag_latest=${{ github.ref_name }}-latest" >> $GITHUB_OUTPUT
+
+    - name: Set environment output from git
+      id: env_var
+      run: |
+        if [ "${{ github.ref_name }}" = "master" ]; then
+          echo "env_name=dev" >> $GITHUB_OUTPUT
+        elif [ "${{ github.ref_name }}" = "1.2.x" ]; then
+          echo "env_name=stage" >> $GITHUB_OUTPUT
+        else
+          echo "env_name=Features" >> $GITHUB_OUTPUT
+        fi         
+
+    - name: Check outputs
+      run: |
+        echo "Branch     : ${{ github.ref_name }}"
+        echo "Tags       : ${{ steps.tag_vars.outputs.tag }}, ${{ steps.tag_vars.outputs.tag_latest }}"
+        echo "Environment: ${{ steps.env_var.outputs.env_name }}"
+
+    - name: Build images
+      env:
+        TAG: ${{ steps.tag_vars.outputs.tag }}
+        TAG_LATEST: ${{ steps.tag_vars.outputs.tag_latest }}
+        REGISTRY_PATH: ${{ steps.registry_path.outputs.registry_path }}
+      working-directory: ./dev
+      # use docker-compose build for 1.29.2
+      #     docker compose build for 2.15.0 (with buildkit enabled by default)
+      run: |
+        docker compose version
+        sed -i "s|registry=|registry=${REGISTRY_PATH}|g" .env
+        sed -i "s/latest/${TAG}/g" .env
+        docker compose build
+        sed -i "s/${TAG}/${TAG_LATEST}/g" .env
+        docker compose build
+        docker images
+
+    - name: Log in to the Container registry
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Push images to the Container registry
+      env:
+        TAG: ${{ steps.tag_vars.outputs.tag }}
+        TAG_LATEST: ${{ steps.tag_vars.outputs.tag_latest }}
+      working-directory: ./dev
+      run: |
+        docker-compose push
+        sed -i "s/${TAG_LATEST}/${TAG}/g" .env
+        docker-compose push
+
+  deploy:
+    needs: build
+    # It's not possible to use natively env (e.g. env.ENV_NAME) variable on the runs-on job field (yet?)
+    # for deploy to different environments depending on branch https://github.com/actions/runner/issues/480
+    # That's why we use output from the previous build job
+    # Note: we are using self-hosted runner here 
+    runs-on: ["${{needs.build.outputs.env_name}}"]
+
+    steps:
+    - name: Checkout Repo 
+      uses: actions/checkout@v3
+
+    - name: Deploy 
+      working-directory: ./dev
+      env:
+        TAG_LATEST: ${{ needs.build.outputs.tag_latest }}
+        REGISTRY_PATH: ${{ needs.build.outputs.registry_path }}
+      run: |
+        sed -i "s|registry=|registry=${REGISTRY_PATH}|g" .env
+        sed -i "s/latest/${TAG_LATEST}/g" .env
+        sed -i "s/uwsgi_processes=2/uwsgi_processes=1/g" .env
+        sudo docker-compose stop
+        sudo docker system prune -a -f
+        sudo docker-compose pull
+        HOSTNAME=$HOSTNAME sudo docker-compose -f docker-compose.yml -f docker-compose.env.yml up -d

.github/workflows/Deploy-qa-demo-manual.yml

@@ -0,0 +1,45 @@
+name: Deploy to QA or Demo environment manually
+
+on: 
+  workflow_dispatch:
+    inputs:
+      environment:
+        type: choice
+        description: 'Choose environment to deploy'
+        required: true
+        default: 'qa'
+        options:
+        - qa
+        - demo
+
+env:
+  REGISTRY_PATH: ghcr.io/exadel-inc/compreface/
+
+jobs:
+  deploy:
+    runs-on: ${{ github.event.inputs.environment }}
+
+    steps:
+
+    - name: Set environment output from git
+      id: tag_var
+      run: |
+        if [ "${{ github.event.inputs.environment }}" = "qa" ]; then
+          echo "TAG_LATEST=master-latest" >> $GITHUB_ENV
+        else
+          echo "TAG_LATEST=1.1.x-latest" >> $GITHUB_ENV
+        fi   
+
+    - name: Checkout Repo 
+      uses: actions/checkout@v3
+
+    - name: Deploy 
+      working-directory: ./dev
+      run: |
+        sed -i "s|registry=|registry=${{ env.REGISTRY_PATH }}|g" .env
+        sed -i "s/latest/${{ env.TAG_LATEST }}/g" .env
+        sed -i "s/uwsgi_processes=2/uwsgi_processes=1/g" .env
+        sudo docker-compose stop
+        sudo docker system prune -a -f
+        sudo docker-compose pull
+        HOSTNAME=$HOSTNAME sudo docker-compose -f docker-compose.yml -f docker-compose.env.yml up -d