diff --git a/.dockerignore b/.dockerignore index 205d9a2cbc..191381ee74 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1 @@ -.git -/embedding-calculator/sample_images \ No newline at end of file +.git \ No newline at end of file diff --git a/.env b/.env index bd92cf6138..7a07475f28 100644 --- a/.env +++ b/.env @@ -10,9 +10,17 @@ email_from= email_password= enable_email_server=false save_images_to_db=true -compreface_api_java_options=-Xmx8g -compreface_admin_java_options=-Xmx8g -ADMIN_VERSION=0.5.1 -API_VERSION=0.5.1 -FE_VERSION=0.5.1 -CORE_VERSION=0.5.1 \ No newline at end of file +compreface_api_java_options=-Xmx4g +compreface_admin_java_options=-Xmx1g +max_file_size=5MB +max_request_size=10M +max_detect_size=640 +uwsgi_processes=2 +uwsgi_threads=1 +connection_timeout=10000 +read_timeout=60000 +ADMIN_VERSION=1.2.0 +API_VERSION=1.2.0 +FE_VERSION=1.2.0 +CORE_VERSION=1.2.0 +POSTGRES_VERSION=1.2.0 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 2fcf1c6d00..42f346a272 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -8,9 +8,11 @@ assignees: '' --- **Describe the bug** + A clear and concise description of what the bug is. **To Reproduce** + Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' @@ -18,15 +20,28 @@ Steps to reproduce the behavior: 4. See error **Expected behavior** + A clear and concise description of what you expected to happen. **Screenshots** + If applicable, add screenshots to help explain your problem. **Desktop (please complete the following information):** + - OS: [e.g. iOS] - Browser [e.g. chrome, safari] - Version [e.g. 22] +**Logs** + +Run those commands and attach result to the ticket: + +`docker ps` + +`docker-compose logs` + + **Additional context** + Add any other context about the problem here. diff --git a/.github/workflows/AWS-CompreFace-image.yml b/.github/workflows/AWS-CompreFace-image.yml new file mode 100644 index 0000000000..413599ace6 --- /dev/null +++ b/.github/workflows/AWS-CompreFace-image.yml @@ -0,0 +1,59 @@ +name: (AWS) Create CompreFace image for new release + +on: + workflow_dispatch: + inputs: + release: + description: release zip (e.g., https://github.com/exadel-inc/CompreFace/releases/download/v1.0.0/CompreFace_1.0.0.zip) + required: true + version: + description: version (e.g., 1.0.0) + required: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_KEY_ACCESS }} + aws-region: us-east-1 + - name: Create Security Group + run: | + export SECURITY_GROUP_ID=$(aws ec2 create-security-group --group-name MySecurityGroup --description "My security group" --query 'GroupId' --output text) + aws ec2 wait security-group-exists --group-ids ${SECURITY_GROUP_ID} + aws ec2 authorize-security-group-ingress --group-id ${SECURITY_GROUP_ID} --protocol tcp --port 22 --cidr 0.0.0.0/0 + echo "SECURITY_GROUP_ID=$SECURITY_GROUP_ID" >> $GITHUB_ENV + - name: Run Instance + run: | + echo ${SECURITY_GROUP_ID} + export INSTANCE_ID=$(aws ec2 run-instances --image-id ami-04e612d1108883950 --count 1 --instance-type t2.medium --key-name IharB --security-group-ids ${SECURITY_GROUP_ID} --subnet-id subnet-080dc6a6ed9580c77 --query 'Instances[0].InstanceId' --output text) + aws ec2 wait instance-running --instance-ids ${INSTANCE_ID} + echo "INSTANCE_ID=$INSTANCE_ID" >> $GITHUB_ENV + sleep 10 + - name: Install Release + env: + RELEASE: ${{ github.event.inputs.release }} + SSH_KEY: ${{secrets.SSH_KEY}} + run: | + echo "$SSH_KEY" > private_key && chmod 600 private_key + export INSTANCE_IP_ADDRESS_EXTERNAL=$(aws ec2 describe-instances --instance-id ${INSTANCE_ID} --query 'Reservations[].Instances[].NetworkInterfaces[].Association.PublicIp' --output text) + echo $INSTANCE_IP_ADDRESS_EXTERNAL + ssh -i private_key -oStrictHostKeyChecking=no ec2-user@$INSTANCE_IP_ADDRESS_EXTERNAL "wget -q -O tmp.zip '$RELEASE' && unzip -o tmp.zip && rm tmp.zip && docker-compose stop && docker-compose rm --force && docker image prune -a --force && docker-compose up -d && rm /home/ec2-user/.ssh/authorized_keys && sudo rm /root/.ssh/authorized_keys" + - name: Stop Instance + run: | + aws ec2 stop-instances --instance-ids $INSTANCE_ID + aws ec2 wait instance-stopped --instance-ids $INSTANCE_ID + - name: Create Image + env: + VERSION: ${{ github.event.inputs.version }} + run: | + export IMAGE_ID=$(aws ec2 create-image --instance-id $INSTANCE_ID --name "CompreFace_${VERSION}" --description "CompreFace Base Image" --query 'ImageId' --output text) + echo "CompreFace Base Image id of version ${VERSION} : ${IMAGE_ID}" + aws ec2 wait image-available --image-ids ${IMAGE_ID} + - name: Delete resources + run: | + aws ec2 terminate-instances --instance-ids $INSTANCE_ID + aws ec2 wait instance-terminated --instance-ids $INSTANCE_ID + aws ec2 delete-security-group --group-name MySecurityGroup diff --git "a/.github/workflows/AWS-\320\241ompreface-packer-image.yml" "b/.github/workflows/AWS-\320\241ompreface-packer-image.yml" new file mode 100644 index 0000000000..1d911cfe1b --- /dev/null +++ "b/.github/workflows/AWS-\320\241ompreface-packer-image.yml" @@ -0,0 +1,40 @@ +--- +name: (AWS) Сompreface packer image +on: + workflow_dispatch: +jobs: + packer: + runs-on: ubuntu-latest + name: packer + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_KEY_ACCESS }} + aws-region: us-east-1 + + - name: Initialize Packer Template + uses: hashicorp/packer-github-actions@master + with: + command: init + + - name: Validate Template + uses: hashicorp/packer-github-actions@master + with: + command: validate + arguments: -syntax-only + target: aws-compreface.pkr.hcl + + - name: Build Artifact + uses: hashicorp/packer-github-actions@master + with: + command: build + arguments: "-color=true -on-error=abort" + target: aws-compreface.pkr.hcl + env: + PACKER_LOG: 1 diff --git a/.github/workflows/Azure-Compreface-AIB-image.yml b/.github/workflows/Azure-Compreface-AIB-image.yml new file mode 100644 index 0000000000..0c7ed7d542 --- /dev/null +++ b/.github/workflows/Azure-Compreface-AIB-image.yml @@ -0,0 +1,51 @@ +--- +name: (Azure) Сompreface AIB image +on: + workflow_dispatch: + inputs: + version: + description: Version (e.g., 1.0.0) + required: true +env: + RESOURCE_GROUP_NAME: compreFaceGallery-RG + MANAGED_IDENTITY: compreFace-MI + GALLERY_NAME: compreFaceGallery + IMAGE_NAME: compreFace + VERSION: ${{ github.event.inputs.version }} +jobs: + build-image: + runs-on: ubuntu-latest + steps: + + - name: Login via Az module + uses: azure/login@v1 + with: + creds: ${{secrets.AZURE_CREDENTIALS}} + + - name: Build custom VM image + id: imageBuilder + uses: azure/build-vm-image@v0 + with: + resource-group-name: "${{ env.RESOURCE_GROUP_NAME }}" + managed-identity: "${{ env.MANAGED_IDENTITY }}" + location: 'eastus' + source-os-type: 'linux' + source-image-type: 'PlatformImage' + source-image: Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:latest + vm-size: 'Standard_B2s' + dist-type: 'SharedImageGallery' + dist-resource-id: "/subscriptions/${{secrets.AZURE_SUBSCRIPTION_ID}}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/providers/Microsoft.Compute/galleries/${{ env.GALLERY_NAME }}/images/${{ env.IMAGE_NAME }}/versions/${{ env.VERSION }}" + dist-location: 'eastus' + customizer-script: | + apt update + apt upgrade -y + apt install -y docker.io unzip + docker version + curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + docker-compose version + chmod 666 /var/run/docker.sock + wget -q -O tmp.zip "https://github.com/exadel-inc/CompreFace/releases/download/v${{ env.VERSION }}/CompreFace_${{ env.VERSION }}.zip" && unzip tmp.zip && rm tmp.zip + sed -i "s|8000:|80:|g" docker-compose.yml + docker-compose pull --quiet + docker-compose up -d diff --git a/.github/workflows/Build-Deploy-auto.yml b/.github/workflows/Build-Deploy-auto.yml new file mode 100644 index 0000000000..bf129d211e --- /dev/null +++ b/.github/workflows/Build-Deploy-auto.yml @@ -0,0 +1,115 @@ +name: Build and Deploy CompreFace on push + +on: + push: + branches: + - master + - '1.2.x' + +env: + REGISTRY: ghcr.io + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # Map steps outputs to a job outputs. + # We need to share it between build and deploy jobs. + outputs: + registry_path: ${{ steps.registry_path.outputs.registry_path }} + tag: ${{ steps.tag_vars.outputs.tag }} + tag_latest: ${{ steps.tag_vars.outputs.tag_latest }} + env_name: ${{ steps.env_var.outputs.env_name }} + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Set registry path output + id: registry_path + run: echo "registry_path=${{ env.REGISTRY }}/exadel-inc/compreface/" >> $GITHUB_OUTPUT + + - name: Set tags from git output + id: tag_vars + run: | + echo "tag=${{ github.ref_name }}-$(git rev-parse HEAD | cut -c 1-7 | tr -d '\n')" >> $GITHUB_OUTPUT + echo "tag_latest=${{ github.ref_name }}-latest" >> $GITHUB_OUTPUT + + - name: Set environment output from git + id: env_var + run: | + if [ "${{ github.ref_name }}" = "master" ]; then + echo "env_name=dev" >> $GITHUB_OUTPUT + elif [ "${{ github.ref_name }}" = "1.2.x" ]; then + echo "env_name=stage" >> $GITHUB_OUTPUT + else + echo "env_name=Features" >> $GITHUB_OUTPUT + fi + + - name: Check outputs + run: | + echo "Branch : ${{ github.ref_name }}" + echo "Tags : ${{ steps.tag_vars.outputs.tag }}, ${{ steps.tag_vars.outputs.tag_latest }}" + echo "Environment: ${{ steps.env_var.outputs.env_name }}" + + - name: Build images + env: + TAG: ${{ steps.tag_vars.outputs.tag }} + TAG_LATEST: ${{ steps.tag_vars.outputs.tag_latest }} + REGISTRY_PATH: ${{ steps.registry_path.outputs.registry_path }} + working-directory: ./dev + # use docker-compose build for 1.29.2 + # docker compose build for 2.15.0 (with buildkit enabled by default) + run: | + docker compose version + sed -i "s|registry=|registry=${REGISTRY_PATH}|g" .env + sed -i "s/latest/${TAG}/g" .env + docker compose build + sed -i "s/${TAG}/${TAG_LATEST}/g" .env + docker compose build + docker images + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push images to the Container registry + env: + TAG: ${{ steps.tag_vars.outputs.tag }} + TAG_LATEST: ${{ steps.tag_vars.outputs.tag_latest }} + working-directory: ./dev + run: | + docker-compose push + sed -i "s/${TAG_LATEST}/${TAG}/g" .env + docker-compose push + + deploy: + needs: build + # It's not possible to use natively env (e.g. env.ENV_NAME) variable on the runs-on job field (yet?) + # for deploy to different environments depending on branch https://github.com/actions/runner/issues/480 + # That's why we use output from the previous build job + # Note: we are using self-hosted runner here + runs-on: ["${{needs.build.outputs.env_name}}"] + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Deploy + working-directory: ./dev + env: + TAG_LATEST: ${{ needs.build.outputs.tag_latest }} + REGISTRY_PATH: ${{ needs.build.outputs.registry_path }} + run: | + sed -i "s|registry=|registry=${REGISTRY_PATH}|g" .env + sed -i "s/latest/${TAG_LATEST}/g" .env + sed -i "s/uwsgi_processes=2/uwsgi_processes=1/g" .env + sudo docker-compose stop + sudo docker system prune -a -f + sudo docker-compose pull + HOSTNAME=$HOSTNAME sudo docker-compose -f docker-compose.yml -f docker-compose.env.yml up -d diff --git a/.github/workflows/Deploy-qa-demo-manual.yml b/.github/workflows/Deploy-qa-demo-manual.yml new file mode 100644 index 0000000000..095f52c431 --- /dev/null +++ b/.github/workflows/Deploy-qa-demo-manual.yml @@ -0,0 +1,45 @@ +name: Deploy to QA or Demo environment manually + +on: + workflow_dispatch: + inputs: + environment: + type: choice + description: 'Choose environment to deploy' + required: true + default: 'qa' + options: + - qa + - demo + +env: + REGISTRY_PATH: ghcr.io/exadel-inc/compreface/ + +jobs: + deploy: + runs-on: ${{ github.event.inputs.environment }} + + steps: + + - name: Set environment output from git + id: tag_var + run: | + if [ "${{ github.event.inputs.environment }}" = "qa" ]; then + echo "TAG_LATEST=master-latest" >> $GITHUB_ENV + else + echo "TAG_LATEST=1.1.x-latest" >> $GITHUB_ENV + fi + + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Deploy + working-directory: ./dev + run: | + sed -i "s|registry=|registry=${{ env.REGISTRY_PATH }}|g" .env + sed -i "s/latest/${{ env.TAG_LATEST }}/g" .env + sed -i "s/uwsgi_processes=2/uwsgi_processes=1/g" .env + sudo docker-compose stop + sudo docker system prune -a -f + sudo docker-compose pull + HOSTNAME=$HOSTNAME sudo docker-compose -f docker-compose.yml -f docker-compose.env.yml up -d diff --git a/.github/workflows/Push-Python-to-Dockerhub-gpu.yml b/.github/workflows/Push-Python-to-Dockerhub-gpu.yml deleted file mode 100644 index 08b82c9208..0000000000 --- a/.github/workflows/Push-Python-to-Dockerhub-gpu.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Docker Image-gpu Python push to Dockerhub - -on: - workflow_dispatch: - inputs: - version: - description: Version (e.g., 0.4.1) - required: true -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build the Docker image and push - env: - VERSION: ${{ github.event.inputs.version }} - DOCKER_USER: ${{secrets.DOCKER_HUB_LOGIN}} - DOCKER_PASSWORD: ${{secrets.DOCKER_HUB_PWD}} - DOCKER_REGISTRY: ${{secrets.DOCKER_HUB_LOGIN}}/ - working-directory: ./embedding-calculator/ - run: | - make build-images-gpu - docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - docker push --all-tags ${DOCKER_REGISTRY}compreface-core - diff --git a/.github/workflows/Push-Python-to-Dockerhub.yml b/.github/workflows/Push-Python-to-Dockerhub.yml deleted file mode 100644 index c16010bd4a..0000000000 --- a/.github/workflows/Push-Python-to-Dockerhub.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Docker Image Python push to Dockerhub - -on: - workflow_dispatch: - inputs: - version: - description: Version (e.g., 0.4.1) - required: true -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build the Docker image and push - env: - VERSION: ${{ github.event.inputs.version }} - DOCKER_USER: ${{secrets.DOCKER_HUB_LOGIN}} - DOCKER_PASSWORD: ${{secrets.DOCKER_HUB_PWD}} - DOCKER_REGISTRY: exadel/ - working-directory: ./embedding-calculator/ - run: | - make build-images-cpu - docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}" - docker push --all-tags ${DOCKER_REGISTRY}compreface-core-base - docker push --all-tags ${DOCKER_REGISTRY}compreface-core - diff --git a/.github/workflows/Release-Push-Single-Image-to-Dockerhub.yml b/.github/workflows/Release-Push-Single-Image-to-Dockerhub.yml new file mode 100644 index 0000000000..ba12f23464 --- /dev/null +++ b/.github/workflows/Release-Push-Single-Image-to-Dockerhub.yml @@ -0,0 +1,74 @@ +name: (Release) Build and Push Single image to Production + +on: + workflow_dispatch: + inputs: + version: + description: Version (e.g., 1.0.0) + required: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check Out Repo + uses: actions/checkout@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_LOGIN }} + password: ${{ secrets.DOCKER_HUB_PWD }} + + - name: Build facenet images + env: + VERSION: ${{ github.event.inputs.version }} + APPERY_API_KEY: ${{ secrets.APPERY_API_KEY }} + working-directory: ./custom-builds/Single-Docker-File/ + run: | + docker build -f Dockerfile --build-arg BASE_IMAGE=exadel/compreface-core:${VERSION} --build-arg VERSION=${VERSION} --build-arg APPERY_API_KEY=${APPERY_API_KEY} -t exadel/compreface:latest ./../.. + docker build -f Dockerfile --build-arg BASE_IMAGE=exadel/compreface-core:${VERSION} --build-arg VERSION=${VERSION} --build-arg APPERY_API_KEY=${APPERY_API_KEY} -t exadel/compreface:${VERSION} ./../.. + docker build -f Dockerfile --build-arg BASE_IMAGE=exadel/compreface-core:${VERSION}-facenet --build-arg VERSION=${VERSION} --build-arg APPERY_API_KEY=${APPERY_API_KEY} -t exadel/compreface:${VERSION}-facenet ./../.. + docker images + - name: Push facenet images to Docker Hub + working-directory: ./custom-builds/Single-Docker-File/ + run: | + docker push --all-tags exadel/compreface + + - name: clean docker + working-directory: ./custom-builds/Single-Docker-File/ + run: | + docker system prune --force + + - name: Build insightface cpu images + env: + VERSION: ${{ github.event.inputs.version }} + APPERY_API_KEY: ${{ secrets.APPERY_API_KEY }} + working-directory: ./custom-builds/Single-Docker-File/ + run: | + docker build -f Dockerfile --build-arg BASE_IMAGE=exadel/compreface-core:${VERSION}-mobilenet --build-arg VERSION=${VERSION} --build-arg APPERY_API_KEY=${APPERY_API_KEY} -t exadel/compreface:${VERSION}-mobilenet ./../.. + docker build -f Dockerfile --build-arg BASE_IMAGE=exadel/compreface-core:${VERSION}-arcface-r100 --build-arg VERSION=${VERSION} --build-arg APPERY_API_KEY=${APPERY_API_KEY} -t exadel/compreface:${VERSION}-arcface-r100 ./../.. + docker images + - name: Push insightface cpu images to Docker Hub + working-directory: ./custom-builds/Single-Docker-File/ + run: | + docker push --all-tags exadel/compreface + + - name: clean docker + working-directory: ./custom-builds/Single-Docker-File/ + run: | + docker system prune --force + + - name: Build insightface gpu images + env: + VERSION: ${{ github.event.inputs.version }} + APPERY_API_KEY: ${{ secrets.APPERY_API_KEY }} + working-directory: ./custom-builds/Single-Docker-File/ + run: | + docker build -f Dockerfile --build-arg BASE_IMAGE=exadel/compreface-core:${VERSION}-mobilenet-gpu --build-arg VERSION=${VERSION} --build-arg APPERY_API_KEY=${APPERY_API_KEY} -t exadel/compreface:${VERSION}-mobilenet-gpu ./../.. + docker build -f Dockerfile --build-arg BASE_IMAGE=exadel/compreface-core:${VERSION}-arcface-r100-gpu --build-arg VERSION=${VERSION} --build-arg APPERY_API_KEY=${APPERY_API_KEY} -t exadel/compreface:${VERSION}-arcface-r100-gpu ./../.. + docker images + - name: Push insightface gpu images to Docker Hub + working-directory: ./custom-builds/Single-Docker-File/ + run: | + docker push --all-tags exadel/compreface + diff --git a/.github/workflows/Release-custom.yml b/.github/workflows/Release-custom.yml new file mode 100644 index 0000000000..c8c578de27 --- /dev/null +++ b/.github/workflows/Release-custom.yml @@ -0,0 +1,37 @@ +name: (Release) Build and Push Custom builds + +on: + workflow_dispatch: + inputs: + version: + description: Version (e.g., 1.0.0) + required: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check Out Repo + uses: actions/checkout@v2 + - name: Build images + env: + VERSION: ${{ github.event.inputs.version }} + APPERY_API_KEY: ${{ secrets.APPERY_API_KEY }} + DOCKER_REGISTRY: exadel/ + working-directory: ./embedding-calculator/ + run: | + make build-images + docker images + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_LOGIN }} + password: ${{ secrets.DOCKER_HUB_PWD }} + - name: Push images to Docker Hub + env: + DOCKER_REGISTRY: exadel/ + working-directory: ./embedding-calculator/ + run: | + docker push --all-tags ${DOCKER_REGISTRY}compreface-core-base + docker push --all-tags ${DOCKER_REGISTRY}compreface-core + + diff --git a/.github/workflows/Release-default.yml b/.github/workflows/Release-default.yml new file mode 100644 index 0000000000..55f3fb82bc --- /dev/null +++ b/.github/workflows/Release-default.yml @@ -0,0 +1,38 @@ +name: (Release) Build and Push Default CompreFace version + +on: + workflow_dispatch: + inputs: + version: + description: Version (e.g., 1.0.0) + required: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check Out Repo + uses: actions/checkout@v2 + - name: Build images + env: + VERSION: ${{ github.event.inputs.version }} + APPERY_API_KEY: ${{ secrets.APPERY_API_KEY }} + working-directory: ./dev/ + run: | + sed -i 's/registry=/registry=exadel\//g' .env + docker-compose build --build-arg APPERY_API_KEY=${APPERY_API_KEY} + sed -i 's/latest/${VERSION}/g' .env + docker-compose build --build-arg APPERY_API_KEY=${APPERY_API_KEY} + docker images + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_LOGIN }} + password: ${{ secrets.DOCKER_HUB_PWD }} + - name: Push images to Docker Hub + working-directory: ./dev/ + env: + VERSION: ${{ github.event.inputs.version }} + run: | + docker-compose push + sed -i 's/${VERSION}/latest/g' .env + docker-compose push \ No newline at end of file diff --git a/.github/workflows/Saving-visitors-statistics-to-S3.yml b/.github/workflows/Saving-visitors-statistics-to-S3.yml index 9266f93ee5..59b2435e9c 100644 --- a/.github/workflows/Saving-visitors-statistics-to-S3.yml +++ b/.github/workflows/Saving-visitors-statistics-to-S3.yml @@ -1,9 +1,9 @@ name: Statistics to s3 master -on: - schedule: - # runs once a every day - - cron: "00 8 * * *" +on: workflow_dispatch +# schedule: +# # runs once a every day +# - cron: "00 8 * * *" jobs: traffic: @@ -34,4 +34,4 @@ jobs: AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - SOURCE_DIR: 'traffic' \ No newline at end of file + SOURCE_DIR: 'traffic' diff --git a/.github/workflows/load-tests-k6.yml b/.github/workflows/load-tests-k6.yml new file mode 100644 index 0000000000..2cc82309d2 --- /dev/null +++ b/.github/workflows/load-tests-k6.yml @@ -0,0 +1,66 @@ +name: Load Tests K6 + +on: + workflow_dispatch: + inputs: + environment: + type: choice + description: 'Select the environment under tests' + required: true + default: 'dev' + options: + - dev + - qa + - demo + - stage + vus: + type: string + description: 'Virtual users count' + default: '1' + iterations: + type: string + description: 'Number of iterations' + default: '1' + duration: + type: string + description: 'Time of test execution, [s,m]' + default: '2m' + +jobs: + pre_run: + runs-on: ${{ github.event.inputs.environment }} + outputs: + tests_hostname: ${{ steps.tests_hostname.outputs.tests_hostname }} + steps: + - name: Get under tests environment hostname + id: tests_hostname + run: echo "tests_hostname=$HOSTNAME" >> $GITHUB_OUTPUT + + run_tests: + needs: pre_run + runs-on: devops + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Run Grafana + working-directory: ./load-tests/grafana + run: sudo docker-compose up -d + + - name: K6 tests build and run + working-directory: ./load-tests/docker + env: + TESTS_HOSTNAME: ${{ needs.pre_run.outputs.tests_hostname }} + VUS: ${{ github.event.inputs.vus }} + ITERATIONS: ${{ github.event.inputs.iterations }} + DURATION: ${{ github.event.inputs.duration }} + run: | + sudo docker build -t k6tests . + sudo docker run --rm \ + -e VUS="${VUS}" \ + -e ITERATIONS="${ITERATIONS}" \ + -e DURATION="${DURATION}" \ + -e HOSTNAME="https://${TESTS_HOSTNAME}" \ + -e INFLUXDB_HOSTNAME="http://$(hostname):8086" \ + -e DB_CONNECTION_STRING="user=postgres password=postgres port=6432 dbname=frs host=${TESTS_HOSTNAME} sslmode=disable" \ + k6tests diff --git a/.github/workflows/unit-tests-on-maven.yml b/.github/workflows/unit-tests-on-maven.yml new file mode 100644 index 0000000000..6fef07dfe6 --- /dev/null +++ b/.github/workflows/unit-tests-on-maven.yml @@ -0,0 +1,31 @@ +# This workflow will run a Unit tests on project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +name: Unit tests on Maven + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + tests: + + runs-on: ubuntu-latest + + steps: + + - uses: actions/checkout@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + + - name: Unit tests with Maven + working-directory: ./java + run: mvn clean test + diff --git a/.github/workflows/unit-tests-on-python.yml b/.github/workflows/unit-tests-on-python.yml index c31fe71c31..2207fa5886 100644 --- a/.github/workflows/unit-tests-on-python.yml +++ b/.github/workflows/unit-tests-on-python.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.8] steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 8a86695e48..7789bbdce4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ build/ ### VS Code ### -.vscode/ \ No newline at end of file +.vscode/ diff --git a/README.md b/README.md index 1ed9667e12..2e40200725 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ -

CompreFace is a free and open-source face recognition system from Exadel

+

Exadel CompreFace is a leading free and open-source face recognition system

- angular-logo + angular-logo
- CompreFace can be easily integrated into any system without prior machine learning skills. CompreFace provides REST API for face -recognition, face verification, face detection, landmark detection, age, and gender recognition and is easily deployed with docker + Exadel CompreFace is a free and open-source face recognition service that can be easily integrated into any system without prior machine learning skills. + CompreFace provides REST API for face recognition, face verification, face detection, landmark detection, mask detection, head pose detection, age, and gender recognition and is easily deployed with docker.

@@ -42,57 +42,126 @@ recognition, face verification, face detection, landmark detection, age, and gen * [Overview](#overview) * [Screenshots](#screenshots) + * [Video tutorials](#videos) * [News and updates](#news-and-updates) * [Features](#features) + * [Functionalities](#functionalities) * [Getting Started with CompreFace](#getting-started-with-compreface) * [CompreFace SDKs](#compreface-sdks) * [Documentation](/docs) * [How to Use CompreFace](/docs/How-to-Use-CompreFace.md) * [Face Services and Plugins](/docs/Face-services-and-plugins.md) * [Rest API Description](/docs/Rest-API-description.md) + * [Postman documentation and collection](https://documenter.getpostman.com/view/17578263/UUxzAnde) * [Face Recognition Similarity Threshold](/docs/Face-Recognition-Similarity-Threshold.md) * [Configuration](/docs/Configuration.md) * [Architecture and Scalability](/docs/Architecture-and-scalability.md) * [Custom Builds](/docs/Custom-builds.md) * [Face data migration](/docs/Face-data-migration.md) * [User Roles System](/docs/User-Roles-System.md) + * [Face Mask Detection Plugin](/docs/Mask-detection-plugin.md) + * [Kubernetes configuration](https://github.com/exadel-inc/compreface-kubernetes) * [Gathering Anonymous Statistics](/docs/Gathering-anonymous-statistics.md) + * [Installation Options](/docs/Installation-options.md) * [Contributing](#contributing) * [License info](#license-info) - # Overview -CompreFace is a free and open-source face detection and recognition GitHub project. Essentially, it is a docker-based application that can be used as a standalone server or deployed in the cloud. You don’t need prior machine learning skills to set up and use CompreFace. +Exadel CompreFace is a free and open-source face recognition GitHub project. +Essentially, it is a docker-based application that can be used as a standalone server or deployed in the cloud. +You don’t need prior machine learning skills to set up and use CompreFace. -CompreFace provides REST API for face recognition, face verification, face detection, landmark detection, age, and gender recognition. The solution also features a role management system that allows you to easily control who has access to your Face Recognition Services. +The system provides REST API for face recognition, face verification, face detection, landmark detection, mask detection, head pose detection, age, and gender recognition. +The solution also features a role management system that allows you to easily control who has access to your Face Recognition Services. -CompreFace is delivered as a docker-compose config and supports different models that work on CPU and GPU. Our solution is based on state-of-the-art methods and libraries like FaceNet and InsightFace. +CompreFace is delivered as a docker-compose config and supports different models that work on CPU and GPU. +Our solution is based on state-of-the-art methods and libraries like FaceNet and InsightFace. # Screenshots + +

+ + +

+ +
+ More Screenshots + + +

+ + +

+

+ + +

+ +
+ +# Videos + +

+ + CompreFace Face Detection Demo + + + CompreFace Appery.io Demo + +

+ +
+ More Videos + +

-compreface-test-page -compreface-main-page + + CompreFace .NET SDK Demo + + + CompreFace JavaScript SDK Demo +

+
+ # News and updates -[Subscribe](https://exadel-7026941.hs-sites.com/en/en/compreface-news-and-updates) to CompreFace News and Updates to never miss new features and product improvements. +[Subscribe](https://info.exadel.com/en/compreface-news-and-updates) to CompreFace News and Updates to never miss new features and product improvements. # Features - The system can accurately identify people even when it has only “seen” their photo once. Technology-wise, CompreFace has several advantages over similar free face recognition solutions. CompreFace: -- Supports many face recognition services: face identification, face verification, face detection, landmark detection, and age and -gender recognition - Supports both CPU and GPU and is easy to scale up - Is open source and self-hosted, which gives you additional guarantees for data security - Can be deployed either in the cloud or on premises - Can be set up and used without machine learning expertise - Uses FaceNet and InsightFace libraries, which use state-of-the-art face recognition methods -- Features a UI panel for convenient user roles and access management - Starts quickly with just one docker command +# Functionalities + +- Supports many face recognition services: + - [face detection](/docs/Face-services-and-plugins.md#face-detection) + - [face recognition](/docs/Face-services-and-plugins.md#face-recognition) + - [face verification](/docs/Face-services-and-plugins.md#face-verification) + - [landmark detection plugin](/docs/Face-services-and-plugins.md#face-plugins) + - [age recognition plugin](/docs/Face-services-and-plugins.md#face-plugins) + - [gender recognition plugin](/docs/Face-services-and-plugins.md#face-plugins) + - [face mask detection plugin](/docs/Face-services-and-plugins.md#face-plugins) + - [head pose plugin](/docs/Face-services-and-plugins.md#face-plugins) +- Use the CompreFace UI panel for convenient user roles and access management # Getting Started with CompreFace @@ -127,9 +196,11 @@ Follow this [link](/dev) # CompreFace SDKs -| SDK | Repository | -| ---------- | ------ | -| JavaScript | https://github.com/exadel-inc/compreface-javascript-sdk | +| SDK | Repository | +|------------|---------------------------------------------------------| +| JavaScript | https://github.com/exadel-inc/compreface-javascript-sdk | +| Python | https://github.com/exadel-inc/compreface-python-sdk | +| .NET | https://github.com/exadel-inc/compreface-net-sdk | # Documentation @@ -149,6 +220,6 @@ We want to improve our open-source face recognition solution, so your contributi For more information, visit our [contributing](CONTRIBUTING.md) guide, or create a [discussion](https://github.com/exadel-inc/CompreFace/discussions). -# License info +# License info CompreFace is open-source real-time facial recognition software released under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0.html). diff --git a/aws-compreface.pkr.hcl b/aws-compreface.pkr.hcl new file mode 100644 index 0000000000..3084ad1ef0 --- /dev/null +++ b/aws-compreface.pkr.hcl @@ -0,0 +1,45 @@ +packer { + required_plugins { + amazon = { + version = ">= 0.0.1" + source = "github.com/hashicorp/amazon" + } + } +} + +source "amazon-ebs" "compreface" { + ami_name = "compreface_image_1.1.0" + instance_type = "t2.medium" + region = "us-east-1" + source_ami_filter { + filters = { + name = "amzn2-ami-kernel-5.10-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["137112412989"] + } + ssh_username = "ec2-user" +} + +build { + name = "packer" + sources = [ + "source.amazon-ebs.compreface" + ] + provisioner "shell" { + inline = [ + "sudo amazon-linux-extras install docker", + "sudo service docker start", + "sudo usermod -a -G docker ec2-user", + "sudo chkconfig docker on", + "sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose", + "sudo chmod +x /usr/local/bin/docker-compose", + "docker-compose version", + "wget -q -O tmp.zip 'https://github.com/exadel-inc/CompreFace/releases/download/v1.1.0/CompreFace_1.1.0.zip' && unzip tmp.zip && rm tmp.zip && rm /home/ec2-user/.ssh/authorized_keys && sudo rm /root/.ssh/authorized_keys", + "sudo chmod 666 /var/run/docker.sock", + "docker-compose up -d" + ] + } +} diff --git a/custom-builds/FaceNet-tpu/.env b/custom-builds/FaceNet-tpu/.env new file mode 100644 index 0000000000..e8e21667fa --- /dev/null +++ b/custom-builds/FaceNet-tpu/.env @@ -0,0 +1,26 @@ +registry=exadel/ +postgres_username=postgres +postgres_password=postgres +postgres_db=frs +postgres_domain=compreface-postgres-db +postgres_port=5432 +email_host=smtp.gmail.com +email_username= +email_from= +email_password= +enable_email_server=false +save_images_to_db=true +compreface_api_java_options=-Xmx8g +compreface_admin_java_options=-Xmx8g +max_file_size=5MB +max_request_size=10M +max_detect_size=640 +uwsgi_processes=2 +uwsgi_threads=1 +connection_timeout=10000 +read_timeout=60000 +ADMIN_VERSION=1.2.0 +API_VERSION=1.2.0 +FE_VERSION=1.2.0 +CORE_VERSION=1.2.0-facenet-tpu +POSTGRES_VERSION=1.2.0 diff --git a/custom-builds/FaceNet-tpu/docker-compose.yml b/custom-builds/FaceNet-tpu/docker-compose.yml new file mode 100644 index 0000000000..2d4aa6ad3c --- /dev/null +++ b/custom-builds/FaceNet-tpu/docker-compose.yml @@ -0,0 +1,89 @@ +version: '3.4' + +volumes: + postgres-data: + +services: + compreface-postgres-db: + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always + container_name: "compreface-postgres-db" + environment: + - POSTGRES_USER=${postgres_username} + - POSTGRES_PASSWORD=${postgres_password} + - POSTGRES_DB=${postgres_db} + volumes: + - postgres-data:/var/lib/postgresql/data + + compreface-admin: + image: ${registry}compreface-admin:${ADMIN_VERSION} + restart: always + container_name: "compreface-admin" + environment: + - POSTGRES_USER=${postgres_username} + - POSTGRES_PASSWORD=${postgres_password} + - POSTGRES_URL=jdbc:postgresql://${postgres_domain}:${postgres_port}/${postgres_db} + - SPRING_PROFILES_ACTIVE=dev + - ENABLE_EMAIL_SERVER=${enable_email_server} + - EMAIL_HOST=${email_host} + - EMAIL_USERNAME=${email_username} + - EMAIL_FROM=${email_from} + - EMAIL_PASSWORD=${email_password} + - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + depends_on: + - compreface-postgres-db + - compreface-api + + compreface-api: + image: ${registry}compreface-api:${API_VERSION} + restart: always + container_name: "compreface-api" + depends_on: + - compreface-postgres-db + environment: + - POSTGRES_USER=${postgres_username} + - POSTGRES_PASSWORD=${postgres_password} + - POSTGRES_URL=jdbc:postgresql://${postgres_domain}:${postgres_port}/${postgres_db} + - SPRING_PROFILES_ACTIVE=dev + - API_JAVA_OPTS=${compreface_api_java_options} + - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} + + compreface-fe: + image: ${registry}compreface-fe:${FE_VERSION} + restart: always + container_name: "compreface-ui" + ports: + - "8000:80" + depends_on: + - compreface-api + - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms + + compreface-core: + image: ${registry}compreface-core:${CORE_VERSION} + restart: always + container_name: "compreface-core" + environment: + - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-2} + - UWSGI_THREADS=${uwsgi_threads:-1} + healthcheck: + test: curl --fail http://localhost:3000/healthcheck || exit 1 + interval: 10s + retries: 0 + start_period: 0s + timeout: 1s + privileged: true + devices: + - /dev/bus/usb + diff --git a/custom-builds/FaceNet/.env b/custom-builds/FaceNet/.env index fd87b5ac6d..7e69c10384 100644 --- a/custom-builds/FaceNet/.env +++ b/custom-builds/FaceNet/.env @@ -12,7 +12,15 @@ enable_email_server=false save_images_to_db=true compreface_api_java_options=-Xmx8g compreface_admin_java_options=-Xmx8g -ADMIN_VERSION=0.5.1 -API_VERSION=0.5.1 -FE_VERSION=0.5.1 -CORE_VERSION=0.5.1-facenet \ No newline at end of file +max_file_size=5MB +max_request_size=10M +max_detect_size=640 +uwsgi_processes=2 +uwsgi_threads=1 +connection_timeout=10000 +read_timeout=60000 +ADMIN_VERSION=1.2.0 +API_VERSION=1.2.0 +FE_VERSION=1.2.0 +CORE_VERSION=1.2.0-facenet +POSTGRES_VERSION=1.2.0 diff --git a/custom-builds/FaceNet/docker-compose.yml b/custom-builds/FaceNet/docker-compose.yml index 4b5992694f..e6c3670149 100644 --- a/custom-builds/FaceNet/docker-compose.yml +++ b/custom-builds/FaceNet/docker-compose.yml @@ -5,7 +5,8 @@ volumes: services: compreface-postgres-db: - image: postgres:11.5 + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always container_name: "compreface-postgres-db" environment: - POSTGRES_USER=${postgres_username} @@ -16,6 +17,7 @@ services: compreface-admin: image: ${registry}compreface-admin:${ADMIN_VERSION} + restart: always container_name: "compreface-admin" environment: - POSTGRES_USER=${postgres_username} @@ -28,12 +30,15 @@ services: - EMAIL_FROM=${email_from} - EMAIL_PASSWORD=${email_password} - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B depends_on: - compreface-postgres-db - compreface-api compreface-api: image: ${registry}compreface-api:${API_VERSION} + restart: always container_name: "compreface-api" depends_on: - compreface-postgres-db @@ -44,18 +49,32 @@ services: - SPRING_PROFILES_ACTIVE=dev - API_JAVA_OPTS=${compreface_api_java_options} - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} compreface-fe: image: ${registry}compreface-fe:${FE_VERSION} + restart: always container_name: "compreface-ui" ports: - "8000:80" depends_on: - compreface-api - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms compreface-core: image: ${registry}compreface-core:${CORE_VERSION} + restart: always container_name: "compreface-core" environment: - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-2} + - UWSGI_THREADS=${uwsgi_threads:-1} + diff --git a/custom-builds/Mobilenet-gpu/.env b/custom-builds/Mobilenet-gpu/.env index 39ce75aa97..43fadd0ae5 100644 --- a/custom-builds/Mobilenet-gpu/.env +++ b/custom-builds/Mobilenet-gpu/.env @@ -12,7 +12,15 @@ enable_email_server=false save_images_to_db=true compreface_api_java_options=-Xmx8g compreface_admin_java_options=-Xmx8g -ADMIN_VERSION=0.5.1 -API_VERSION=0.5.1 -FE_VERSION=0.5.1 -CORE_VERSION=0.5.1-mobilenet-gpu +max_file_size=5MB +max_request_size=10M +max_detect_size=640 +uwsgi_processes=1 +uwsgi_threads=1 +connection_timeout=10000 +read_timeout=60000 +ADMIN_VERSION=1.2.0 +API_VERSION=1.2.0 +FE_VERSION=1.2.0 +CORE_VERSION=1.2.0-mobilenet-gpu +POSTGRES_VERSION=1.2.0 diff --git a/custom-builds/Mobilenet-gpu/docker-compose.yml b/custom-builds/Mobilenet-gpu/docker-compose.yml index 3bdcbee1d9..b46776bd59 100644 --- a/custom-builds/Mobilenet-gpu/docker-compose.yml +++ b/custom-builds/Mobilenet-gpu/docker-compose.yml @@ -5,7 +5,8 @@ volumes: services: compreface-postgres-db: - image: postgres:11.5 + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always container_name: "compreface-postgres-db" environment: - POSTGRES_USER=${postgres_username} @@ -16,6 +17,7 @@ services: compreface-admin: image: ${registry}compreface-admin:${ADMIN_VERSION} + restart: always container_name: "compreface-admin" environment: - POSTGRES_USER=${postgres_username} @@ -28,12 +30,15 @@ services: - EMAIL_FROM=${email_from} - EMAIL_PASSWORD=${email_password} - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B depends_on: - compreface-postgres-db - compreface-api compreface-api: image: ${registry}compreface-api:${API_VERSION} + restart: always container_name: "compreface-api" depends_on: - compreface-postgres-db @@ -44,19 +49,33 @@ services: - SPRING_PROFILES_ACTIVE=dev - API_JAVA_OPTS=${compreface_api_java_options} - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} compreface-fe: image: ${registry}compreface-fe:${FE_VERSION} + restart: always container_name: "compreface-ui" ports: - "8000:80" depends_on: - compreface-api - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms compreface-core: image: ${registry}compreface-core:${CORE_VERSION} + restart: always container_name: "compreface-core" runtime: nvidia environment: - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-1} + - UWSGI_THREADS=${uwsgi_threads:-1} + diff --git a/custom-builds/Mobilenet/.env b/custom-builds/Mobilenet/.env index cb4bce8bb2..cf3197aa9b 100644 --- a/custom-builds/Mobilenet/.env +++ b/custom-builds/Mobilenet/.env @@ -12,7 +12,15 @@ enable_email_server=false save_images_to_db=true compreface_api_java_options=-Xmx8g compreface_admin_java_options=-Xmx8g -ADMIN_VERSION=0.5.1 -API_VERSION=0.5.1 -FE_VERSION=0.5.1 -CORE_VERSION=0.5.1-mobilenet +max_file_size=5MB +max_request_size=10M +max_detect_size=640 +uwsgi_processes=2 +uwsgi_threads=1 +connection_timeout=10000 +read_timeout=60000 +ADMIN_VERSION=1.2.0 +API_VERSION=1.2.0 +FE_VERSION=1.2.0 +CORE_VERSION=1.2.0-mobilenet +POSTGRES_VERSION=1.2.0 diff --git a/custom-builds/Mobilenet/docker-compose.yml b/custom-builds/Mobilenet/docker-compose.yml index 4b5992694f..e6c3670149 100644 --- a/custom-builds/Mobilenet/docker-compose.yml +++ b/custom-builds/Mobilenet/docker-compose.yml @@ -5,7 +5,8 @@ volumes: services: compreface-postgres-db: - image: postgres:11.5 + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always container_name: "compreface-postgres-db" environment: - POSTGRES_USER=${postgres_username} @@ -16,6 +17,7 @@ services: compreface-admin: image: ${registry}compreface-admin:${ADMIN_VERSION} + restart: always container_name: "compreface-admin" environment: - POSTGRES_USER=${postgres_username} @@ -28,12 +30,15 @@ services: - EMAIL_FROM=${email_from} - EMAIL_PASSWORD=${email_password} - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B depends_on: - compreface-postgres-db - compreface-api compreface-api: image: ${registry}compreface-api:${API_VERSION} + restart: always container_name: "compreface-api" depends_on: - compreface-postgres-db @@ -44,18 +49,32 @@ services: - SPRING_PROFILES_ACTIVE=dev - API_JAVA_OPTS=${compreface_api_java_options} - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} compreface-fe: image: ${registry}compreface-fe:${FE_VERSION} + restart: always container_name: "compreface-ui" ports: - "8000:80" depends_on: - compreface-api - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms compreface-core: image: ${registry}compreface-core:${CORE_VERSION} + restart: always container_name: "compreface-core" environment: - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-2} + - UWSGI_THREADS=${uwsgi_threads:-1} + diff --git a/custom-builds/README.md b/custom-builds/README.md index 8fe2c9cdd1..01cc713eca 100644 --- a/custom-builds/README.md +++ b/custom-builds/README.md @@ -1,9 +1,10 @@ # List of custom-builds -| Custom-build | Base library | CPU | GPU | Face detection model / accuracy on [WIDER Face (Hard)](https://paperswithcode.com/sota/face-detection-on-wider-face-hard) | Face recognition model / accuracy on [LFW](https://paperswithcode.com/sota/face-verification-on-labeled-faces-in-the) | Age and gender detection | Comment | -| -------------------------- | --------------------------------------------------------- | -------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | ---------------------------------------------- | -| FaceNet (default) | [FaceNet](https://github.com/davidsandberg/facenet) | x86 (AVX instructions) | not supported | MTCNN / 80.9% | FaceNet (20180402-114759) / 99.63% | Custom, the model is taken [here](https://github.com/GilLevi/AgeGenderDeepLearning) | For general purposes. Support CPU without AVX2 | -| Mobilenet | [InsightFace](https://github.com/deepinsight/insightface) | x86 (AVX2 instructions) | not supported | RetinaFace-MobileNet0.25 / 82.5% | MobileFaceNet,ArcFace / 99.50% | InsightFace | The fastest model among CPU only models | -| Mobilenet-gpu | [InsightFace](https://github.com/deepinsight/insightface) | x86 (AVX2 instructions) | GPU (CUDA required) | RetinaFace-MobileNet0.25 / 82.5% | MobileFaceNet,ArcFace / 99.50% | InsightFace | The fastest model | -| SubCenter-ArcFace-r100 | [InsightFace](https://github.com/deepinsight/insightface) | x86 (AVX2 instructions) | not supported | retinaface_r50_v1 / 91.4% | arcface-r100-msfdrop75 / 99.80% | InsightFace | The most accurate model, but the most slow | -| SubCenter-ArcFace-r100-gpu | [InsightFace](https://github.com/deepinsight/insightface) | x86 (AVX2 instructions) | GPU (CUDA required) | retinaface_r50_v1 / 91.4% | arcface-r100-msfdrop75 / 99.80% | InsightFace | The most accurate model | +| Custom-build | Base library | CPU | GPU | Face detection model / accuracy on [WIDER Face (Hard)](https://paperswithcode.com/sota/face-detection-on-wider-face-hard) | Face recognition model / accuracy on [LFW](https://paperswithcode.com/sota/face-verification-on-labeled-faces-in-the) | Age and gender detection | Face mask detection | Comment | +|-------------------------------|-----------------------------------------------------------|-------------------------|---------------------|---------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|--------------------------------------------------|------------------------------------------------| +| FaceNet (default) | [FaceNet](https://github.com/davidsandberg/facenet) | x86 (AVX instructions) | not supported | MTCNN / 80.9% | FaceNet (20180402-114759) / 99.63% | Custom, the model is taken [here](https://github.com/GilLevi/AgeGenderDeepLearning) | [Custom model](../docs/Mask-detection-plugin.md) | For general purposes. Support CPU without AVX2 | +| FaceNet Masked (Experimental) | [FaceNet](https://github.com/davidsandberg/facenet) | x86 (AVX instructions) | not supported | MTCNN / 80.9% | inception_v3_on_mafa_kaggle123 / 98.73% | Custom, the model is taken [here](https://github.com/GilLevi/AgeGenderDeepLearning) | [Custom model](../docs/Mask-detection-plugin.md) | For general purposes. Support CPU without AVX2 | +| Mobilenet | [InsightFace](https://github.com/deepinsight/insightface) | x86 (AVX2 instructions) | not supported | RetinaFace-MobileNet0.25 / 82.5% | MobileFaceNet,ArcFace / 99.50% | InsightFace | [Custom model](../docs/Mask-detection-plugin.md) | The fastest model among CPU only models | +| Mobilenet-gpu | [InsightFace](https://github.com/deepinsight/insightface) | x86 (AVX2 instructions) | GPU (CUDA required) | RetinaFace-MobileNet0.25 / 82.5% | MobileFaceNet,ArcFace / 99.50% | InsightFace | [Custom model](../docs/Mask-detection-plugin.md) | The fastest model | +| SubCenter-ArcFace-r100 | [InsightFace](https://github.com/deepinsight/insightface) | x86 (AVX2 instructions) | not supported | retinaface_r50_v1 / 91.4% | arcface-r100-msfdrop75 / 99.80% | InsightFace | [Custom model](../docs/Mask-detection-plugin.md) | The most accurate model, but the most slow | +| SubCenter-ArcFace-r100-gpu | [InsightFace](https://github.com/deepinsight/insightface) | x86 (AVX2 instructions) | GPU (CUDA required) | retinaface_r50_v1 / 91.4% | arcface-r100-msfdrop75 / 99.80% | InsightFace | [Custom model](../docs/Mask-detection-plugin.md) | The most accurate model | diff --git a/custom-builds/Single-Docker-File/Dockerfile b/custom-builds/Single-Docker-File/Dockerfile new file mode 100644 index 0000000000..8a99c3ef12 --- /dev/null +++ b/custom-builds/Single-Docker-File/Dockerfile @@ -0,0 +1,107 @@ +ARG BASE_IMAGE=exadel/compreface-core:latest +ARG VERSION=latest + +################# init images start #################### +FROM exadel/compreface-postgres-db:${VERSION} as postgres_db +FROM exadel/compreface-admin:${VERSION} as admin +FROM exadel/compreface-api:${VERSION} as api +FROM exadel/compreface-fe:${VERSION} as fe +################# init images end #################### + +################# compreface-core start #################### +FROM ${BASE_IMAGE} + +ENV UWSGI_PROCESSES=2 +ENV UWSGI_THREADS=1 +################# compreface-core end #################### + +ENV TZ=America/Los_Angeles +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +################# compreface-postgres-db start #################### +ENV POSTGRES_USER=compreface +ENV POSTGRES_PASSWORD=M7yfTsBscdqvZs49 +ENV POSTGRES_DB=frs +ENV PGDATA=/var/lib/postgresql/data + +RUN apt-get update && apt-get install -y lsb-release +RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN apt-get update && apt-get install -y postgresql-13 \ + && rm -rf /var/lib/apt/lists/* + +RUN rm /etc/postgresql/13/main/postgresql.conf +COPY custom-builds/Single-Docker-File/postgresql.conf /etc/postgresql/13/main/postgresql.conf +RUN mv /var/lib/postgresql/13/main $PGDATA + +COPY --from=postgres_db /docker-entrypoint-initdb.d/initdb.sql /initdb.sql + +USER postgres + +RUN /etc/init.d/postgresql start && \ + psql --command "CREATE USER ${POSTGRES_USER} WITH SUPERUSER PASSWORD '${POSTGRES_PASSWORD}';" && \ + createdb -O ${POSTGRES_USER} ${POSTGRES_DB} && \ + psql -d ${POSTGRES_DB} -a -f /initdb.sql + +RUN cp -r $PGDATA /var/lib/postgresql/default + +USER root + +################# compreface-postgres-db end #################### + +################# compreface-admin start #################### + +ENV POSTGRES_URL=jdbc:postgresql://localhost:5432/${POSTGRES_DB} +ENV SPRING_PROFILES_ACTIVE=dev +ENV ENABLE_EMAIL_SERVER=false +ENV ADMIN_JAVA_OPTS=-Xmx1g +ENV CRUD_PORT=8081 +ENV PYTHON_URL=http://localhost:3000 +ENV MAX_FILE_SIZE=5MB +ENV MAX_REQUEST_SIZE=10MB + +ENV JAVA_HOME=/opt/java/openjdk +COPY --from=eclipse-temurin:17.0.8_7-jdk-jammy $JAVA_HOME $JAVA_HOME +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +COPY --from=admin /home/app.jar /app/admin/app.jar +ARG APPERY_API_KEY +ENV APPERY_API_KEY ${APPERY_API_KEY} + +################# compreface-admin end #################### + +################# compreface-api start #################### + +ENV API_JAVA_OPTS=-Xmx4g +ENV SAVE_IMAGES_TO_DB=true +ENV API_PORT=8080 +ENV CONNECTION_TIMEOUT=10000 +ENV READ_TIMEOUT=60000 + +COPY --from=api /home/app.jar /app/api/app.jar + +################# compreface-api end #################### + +################# compreface-fe start #################### + +RUN apt-get update && apt-get install -y nginx \ + && rm -rf /var/lib/apt/lists/* +RUN adduser --system --no-create-home --shell /bin/false --group --disabled-login nginx + +USER nginx + +COPY --from=fe /usr/share/nginx/html /usr/share/nginx/html +COPY --from=fe /etc/nginx/ /etc/nginx/ +COPY custom-builds/Single-Docker-File/nginx.conf /etc/nginx/conf.d/nginx.conf + +USER root +################# compreface-fe end #################### + +################# supervisord #################### +RUN apt-get update && apt-get install -y supervisor mc && rm -rf /var/lib/apt/lists/* +RUN mkdir -p /var/log/supervisor +COPY custom-builds/Single-Docker-File/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY custom-builds/Single-Docker-File/startup.sh /startup.sh +RUN chmod +x /startup.sh + +CMD ["/usr/bin/supervisord"] diff --git a/custom-builds/Single-Docker-File/nginx.conf b/custom-builds/Single-Docker-File/nginx.conf new file mode 100644 index 0000000000..8542683800 --- /dev/null +++ b/custom-builds/Single-Docker-File/nginx.conf @@ -0,0 +1,75 @@ +upstream frsadmin { + server localhost:8081 fail_timeout=10s max_fails=5; +} + +upstream frsapi { + server localhost:8080 fail_timeout=10s max_fails=5; +} + +upstream frscore { + server localhost:3000 fail_timeout=10s max_fails=5; +} + +server { + listen 80; + error_log stderr; + access_log /dev/stdout; + server_name ui; + + client_max_body_size 10M; + + location / { + root /usr/share/nginx/html/; + index index.html; + try_files $uri $uri/ /index.html =404; + } + + location /admin/ { + proxy_pass http://frsadmin/admin/; + } + + location /api/v1/ { + + proxy_read_timeout 60000ms; + proxy_connect_timeout 10000ms; + + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key' always; + + proxy_pass http://frsapi/api/v1/; + } + + location /core/ { + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key' always; + + proxy_pass http://frscore/; + } + + location ~ ^/(api|admin)/(swagger-ui.html|webjars|swagger-resources|v2/api-docs)(.*) { + proxy_set_header 'Host' $http_host; + proxy_pass http://frs$1/$2$3$is_args$args; + } +} diff --git a/custom-builds/Single-Docker-File/postgresql.conf b/custom-builds/Single-Docker-File/postgresql.conf new file mode 100644 index 0000000000..a94655b37f --- /dev/null +++ b/custom-builds/Single-Docker-File/postgresql.conf @@ -0,0 +1,785 @@ +# CompreFace changes: +# 1. Changed `data_directory`, so it will always link to `/var/lib/postgresql/data` and do not depend on postgres version. + +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: B = bytes Time units: us = microseconds +# kB = kilobytes ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +data_directory = '/var/lib/postgresql/data' # use data in another directory + # (change requires restart) +hba_file = '/etc/postgresql/13/main/pg_hba.conf' # host-based authentication file + # (change requires restart) +ident_file = '/etc/postgresql/13/main/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +external_pid_file = '/var/run/postgresql/13-main.pid' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +#listen_addresses = 'localhost' # what IP address(es) to listen on; + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +port = 5432 # (change requires restart) +max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +unix_socket_directories = '/var/run/postgresql' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - TCP settings - +# see "man tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default +#tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; + # 0 selects the system default + +# - Authentication - + +#authentication_timeout = 1min # 1s-600s +#password_encryption = md5 # md5 or scram-sha-256 +#db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab' +#krb_caseins_users = off + +# - SSL - + +ssl = on +#ssl_ca_file = '' +ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' +#ssl_crl_file = '' +ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +#ssl_prefer_server_ciphers = on +#ssl_ecdh_curve = 'prime256v1' +#ssl_min_protocol_version = 'TLSv1.2' +#ssl_max_protocol_version = '' +#ssl_dh_params_file = '' +#ssl_passphrase_command = '' +#ssl_passphrase_command_supports_reload = off + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +shared_buffers = 128MB # min 128kB + # (change requires restart) +#huge_pages = try # on, off, or try + # (change requires restart) +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +#work_mem = 4MB # min 64kB +#hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem +#maintenance_work_mem = 64MB # min 1MB +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#logical_decoding_work_mem = 64MB # min 64kB +#max_stack_depth = 2MB # min 100kB +#shared_memory_type = mmap # the default is the first option + # supported by the operating system: + # mmap + # sysv + # windows + # (change requires restart) +dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # (change requires restart) + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space + # in kilobytes, or -1 for no limit + +# - Kernel Resources - + +#max_files_per_process = 1000 # min 64 + # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 10 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 512kB # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_maintenance_workers = 2 # taken from max_parallel_workers +#max_parallel_workers_per_gather = 2 # taken from max_parallel_workers +#parallel_leader_participation = on +#max_parallel_workers = 8 # maximum number of max_worker_processes that + # can be used in parallel operations +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate + # (change requires restart) +#backend_flush_after = 0 # measured in pages, 0 disables + + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +#wal_level = replica # minimal, replica, or logical + # (change requires restart) +#fsync = on # flush data to disk for crash safety + # (turning this off can cause + # unrecoverable data corruption) +#synchronous_commit = on # synchronization level; + # off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux and FreeBSD) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_compression = off # enable compression of full-page writes +#wal_log_hints = off # also do full page writes of non-critical updates + # (change requires restart) +#wal_init_zero = on # zero-fill new WAL files +#wal_recycle = on # recycle WAL files +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables +#wal_skip_threshold = 2MB + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +max_wal_size = 1GB +min_wal_size = 80MB +#checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 +#checkpoint_flush_after = 256kB # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always + # (change requires restart) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + +# - Archive Recovery - + +# These are only used in recovery mode. + +#restore_command = '' # command to use to restore an archived logfile segment + # placeholders: %p = path of file to restore + # %f = file name only + # e.g. 'cp /mnt/server/archivedir/%f %p' + # (change requires restart) +#archive_cleanup_command = '' # command to execute at every restartpoint +#recovery_end_command = '' # command to execute at completion of recovery + +# - Recovery Target - + +# Set these only when performing a targeted recovery. + +#recovery_target = '' # 'immediate' to end recovery as soon as a + # consistent state is reached + # (change requires restart) +#recovery_target_name = '' # the named restore point to which recovery will proceed + # (change requires restart) +#recovery_target_time = '' # the time stamp up to which recovery will proceed + # (change requires restart) +#recovery_target_xid = '' # the transaction ID up to which recovery will proceed + # (change requires restart) +#recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed + # (change requires restart) +#recovery_target_inclusive = on # Specifies whether to stop: + # just after the specified recovery target (on) + # just before the recovery target (off) + # (change requires restart) +#recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID + # (change requires restart) +#recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' + # (change requires restart) + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Servers - + +# Set these on the master and on any standby that will send replication data. + +#max_wal_senders = 10 # max number of walsender processes + # (change requires restart) +#wal_keep_size = 0 # in megabytes; 0 disables +#max_slot_wal_keep_size = -1 # in megabytes; -1 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables + +#max_replication_slots = 10 # max number of replication slots + # (change requires restart) +#track_commit_timestamp = off # collect timestamp of transaction commit + # (change requires restart) + +# - Master Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # method to choose sync standbys, number of sync standbys, + # and comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a master server. + +#primary_conninfo = '' # connection string to sending server +#primary_slot_name = '' # replication slot on sending server +#promote_trigger_file = '' # file name whose presence ends recovery +#hot_standby = on # "off" disallows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name + # is not set +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from master + # in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to + # retrieve WAL after a failed attempt +#recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery + +# - Subscribers - + +# These settings are ignored on a publisher. + +#max_logical_replication_workers = 4 # taken from max_worker_processes + # (change requires restart) +#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_bitmapscan = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_parallel_append = on +#enable_seqscan = on +#enable_sort = on +#enable_incremental_sort = on +#enable_tidscan = on +#enable_partitionwise_join = off +#enable_partitionwise_aggregate = off +#enable_parallel_hash = on +#enable_partition_pruning = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above + +#jit_above_cost = 100000 # perform JIT compilation if available + # and query more expensive than this; + # -1 disables +#jit_inline_above_cost = 500000 # inline small functions if query is + # more expensive than this; -1 disables +#jit_optimize_above_cost = 500000 # use expensive JIT optimizations if + # query is more expensive than this; + # -1 disables + +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB +#effective_cache_size = 4GB + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses +#force_parallel_mode = off +#jit = on # allow JIT compilation +#plan_cache_mode = auto # auto, force_generic_plan or + # force_custom_plan + + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +#log_destination = 'stderr' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +#logging_collector = off # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +#log_directory = 'log' # directory where log files are written, + # can be absolute or relative to PGDATA +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, + # can include strftime() escapes +#log_file_mode = 0600 # creation mode for log files, + # begin with 0 to use octal notation +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. +#log_rotation_age = 1d # Automatic rotation of logfiles will + # happen after that time. 0 disables. +#log_rotation_size = 10MB # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (win32): +# (change requires restart) +#event_source = 'PostgreSQL' + +# - When to Log - + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + +#log_min_duration_sample = -1 # -1 is disabled, 0 logs a sample of statements + # and their durations, > 0 logs only a sample of + # statements running at least this number + # of milliseconds; + # sample fraction is determined by log_statement_sample_rate + +#log_statement_sample_rate = 1.0 # fraction of logged statements exceeding + # log_min_duration_sample to be logged; + # 1.0 logs all such statements, 0.0 never logs + + +#log_transaction_sample_rate = 0.0 # fraction of transactions whose statements + # are logged regardless of their duration; 1.0 logs all + # statements from all transactions, 0.0 never logs + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +log_line_prefix = '%m [%p] %q%u@%d ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %b = backend type + # %p = process ID + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_parameter_max_length = -1 # when logging statements, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_parameter_max_length_on_error = 0 # when logging an error, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'Etc/UTC' + +#------------------------------------------------------------------------------ +# PROCESS TITLE +#------------------------------------------------------------------------------ + +cluster_name = '13/main' # added to process titles if nonempty + # (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# - Query and Index Statistics Collector - + +#track_activities = on +#track_counts = on +#track_io_timing = off +#track_functions = none # none, pl, all +#track_activity_query_size = 1024 # (change requires restart) +stats_temp_directory = '/var/run/postgresql/13-main.pg_stat_tmp' + + +# - Monitoring - + +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off +#log_statement_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_vacuum_insert_threshold = 1000 # min number of row inserts + # before vacuum; -1 disables insert + # vacuums +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table + # size before insert vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error +#search_path = '"$user", public' # schema names +#row_security = on +#default_tablespace = '' # a tablespace name, '' uses the default +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#default_table_access_method = 'heap' +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_min_age = 50000000 +#vacuum_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_freeze_table_age = 150000000 +#vacuum_cleanup_index_scale_factor = 0.1 # fraction of total number of tuples + # before index cleanup, 0 always performs + # index cleanup +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_fuzzy_search_limit = 0 +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +timezone = 'Etc/UTC' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +#extra_float_digits = 1 # min -15, max 3; any value >0 actually + # selects precise output mode +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb, but they can be changed. +lc_messages = 'C.UTF-8' # locale for system error message + # strings +lc_monetary = 'C.UTF-8' # locale for monetary formatting +lc_numeric = 'C.UTF-8' # locale for number formatting +lc_time = 'C.UTF-8' # locale for time formatting + +# default configuration for text search +default_text_search_config = 'pg_catalog.english' + +# - Shared Library Preloading - + +#shared_preload_libraries = '' # (change requires restart) +#local_preload_libraries = '' +#session_preload_libraries = '' +#jit_provider = 'llvmjit' # JIT library to use + +# - Other Defaults - + +#dynamic_library_path = '$libdir' +#extension_destdir = '' # prepend path when loading extensions + # and shared objects (added by Debian) + + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_relation = -2 # negative values mean + # (max_pred_locks_per_transaction + # / -max_pred_locks_per_relation) - 1 +#max_pred_locks_per_page = 2 # min 0 + + +#------------------------------------------------------------------------------ +# VERSION AND PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#escape_string_warning = on +#lo_compat_privileges = off +#operator_precedence_warning = off +#quote_all_identifiers = off +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? +#data_sync_retry = off # retry or panic on failure to fsync + # data? + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. Note that these are directives, not variable +# assignments, so they can usefully be given more than once. + +include_dir = 'conf.d' # include files ending in '.conf' from + # a directory, e.g., 'conf.d' +#include_if_exists = '...' # include file only if it exists +#include = '...' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here \ No newline at end of file diff --git a/custom-builds/Single-Docker-File/startup.sh b/custom-builds/Single-Docker-File/startup.sh new file mode 100644 index 0000000000..c8c4802ae4 --- /dev/null +++ b/custom-builds/Single-Docker-File/startup.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# EXTERNAL_DB defines if we need to run internal DB +external_db=${EXTERNAL_DB:-false} +if [ "$external_db" = false ] ; then + # restore default data if it was cleared by volume creation + if [ -z "$(ls -A $PGDATA)" ]; then + echo "Postgres directory is empty. Copy default values into it" + cp -r /var/lib/postgresql/default/* $PGDATA + fi + # change permissions in case they were corrupted + chown -R postgres:postgres $PGDATA + chmod 700 $PGDATA + + echo Starting compreface-postgres-db + supervisorctl start compreface-postgres-db +fi + +# wait until DB starts +sleep 10 +echo Starting compreface-admin +supervisorctl start compreface-admin + +# wait until compreface-admin make all migrations +sleep 10 +echo Starting compreface-api +supervisorctl start compreface-api + +# wait until compreface-admin starts +sleep 10 +echo Starting compreface-fe +supervisorctl start compreface-fe \ No newline at end of file diff --git a/custom-builds/Single-Docker-File/supervisord.conf b/custom-builds/Single-Docker-File/supervisord.conf new file mode 100644 index 0000000000..e5f5a53771 --- /dev/null +++ b/custom-builds/Single-Docker-File/supervisord.conf @@ -0,0 +1,78 @@ +[supervisord] +nodaemon=true + +[program:startup] +command=/startup.sh +startsecs=0 +priority=1 +autostart=true +autorestart=false +startretries=1 +stdout_logfile=/dev/fd/2 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:compreface-postgres-db] +command=/usr/lib/postgresql/13/bin/postgres -D /var/lib/postgresql/data -c config_file=/etc/postgresql/13/main/postgresql.conf +user=postgres +startsecs=0 +priority=1 +killasgroup=true +stopasgroup=true +stopsignal=INT +autostart=false +autorestart=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:compreface-core] +command=uwsgi --ini uwsgi.ini +directory=/app/ml +startsecs=0 +priority=2 +killasgroup=true +stopasgroup=true +autostart=true +autorestart=true +stdout_logfile=/dev/fd/2 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:compreface-admin] +command=java %(ENV_ADMIN_JAVA_OPTS)s -jar /app/admin/app.jar +directory=/app/admin +startsecs=0 +priority=2 +killasgroup=true +stopasgroup=true +autostart=false +autorestart=true +stdout_logfile=/dev/fd/2 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:compreface-api] +command=java %(ENV_API_JAVA_OPTS)s -jar /app/api/app.jar +directory=/app/api +startsecs=0 +priority=2 +killasgroup=true +stopasgroup=true +autostart=false +autorestart=true +stdout_logfile=/dev/fd/2 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:compreface-fe] +command=/usr/sbin/nginx -g "daemon off;" +startsecs=0 +priority=100 +killasgroup=true +stopasgroup=true +autostart=false +autorestart=true +stdout_logfile=/dev/fd/2 +stdout_logfile_maxbytes=0 +redirect_stderr=true \ No newline at end of file diff --git a/custom-builds/SubCenter-ArcFace-r100-gpu/.env b/custom-builds/SubCenter-ArcFace-r100-gpu/.env index 25fe2db1d3..ceabb1458e 100644 --- a/custom-builds/SubCenter-ArcFace-r100-gpu/.env +++ b/custom-builds/SubCenter-ArcFace-r100-gpu/.env @@ -12,7 +12,15 @@ enable_email_server=false save_images_to_db=true compreface_api_java_options=-Xmx8g compreface_admin_java_options=-Xmx8g -ADMIN_VERSION=0.5.1 -API_VERSION=0.5.1 -FE_VERSION=0.5.1 -CORE_VERSION=0.5.1-arcface-r100-gpu +max_file_size=5MB +max_request_size=10M +max_detect_size=640 +uwsgi_processes=2 +uwsgi_threads=1 +connection_timeout=10000 +read_timeout=60000 +ADMIN_VERSION=1.2.0 +API_VERSION=1.2.0 +FE_VERSION=1.2.0 +CORE_VERSION=1.2.0-arcface-r100-gpu +POSTGRES_VERSION=1.2.0 diff --git a/custom-builds/SubCenter-ArcFace-r100-gpu/docker-compose.yml b/custom-builds/SubCenter-ArcFace-r100-gpu/docker-compose.yml index 3bdcbee1d9..95f48dae3c 100644 --- a/custom-builds/SubCenter-ArcFace-r100-gpu/docker-compose.yml +++ b/custom-builds/SubCenter-ArcFace-r100-gpu/docker-compose.yml @@ -5,7 +5,8 @@ volumes: services: compreface-postgres-db: - image: postgres:11.5 + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always container_name: "compreface-postgres-db" environment: - POSTGRES_USER=${postgres_username} @@ -16,6 +17,7 @@ services: compreface-admin: image: ${registry}compreface-admin:${ADMIN_VERSION} + restart: always container_name: "compreface-admin" environment: - POSTGRES_USER=${postgres_username} @@ -28,12 +30,15 @@ services: - EMAIL_FROM=${email_from} - EMAIL_PASSWORD=${email_password} - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B depends_on: - compreface-postgres-db - compreface-api compreface-api: image: ${registry}compreface-api:${API_VERSION} + restart: always container_name: "compreface-api" depends_on: - compreface-postgres-db @@ -44,19 +49,33 @@ services: - SPRING_PROFILES_ACTIVE=dev - API_JAVA_OPTS=${compreface_api_java_options} - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} compreface-fe: image: ${registry}compreface-fe:${FE_VERSION} + restart: always container_name: "compreface-ui" ports: - "8000:80" depends_on: - compreface-api - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms compreface-core: image: ${registry}compreface-core:${CORE_VERSION} + restart: always container_name: "compreface-core" runtime: nvidia environment: - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-2} + - UWSGI_THREADS=${uwsgi_threads:-1} + diff --git a/custom-builds/SubCenter-ArcFace-r100/.env b/custom-builds/SubCenter-ArcFace-r100/.env index 34dc6c31fc..e44c43cf38 100644 --- a/custom-builds/SubCenter-ArcFace-r100/.env +++ b/custom-builds/SubCenter-ArcFace-r100/.env @@ -12,7 +12,15 @@ enable_email_server=false save_images_to_db=true compreface_api_java_options=-Xmx8g compreface_admin_java_options=-Xmx8g -ADMIN_VERSION=0.5.1 -API_VERSION=0.5.1 -FE_VERSION=0.5.1 -CORE_VERSION=0.5.1-arcface-r100 \ No newline at end of file +max_file_size=5MB +max_request_size=10M +max_detect_size=640 +uwsgi_processes=2 +uwsgi_threads=1 +connection_timeout=10000 +read_timeout=60000 +ADMIN_VERSION=1.2.0 +API_VERSION=1.2.0 +FE_VERSION=1.2.0 +CORE_VERSION=1.2.0-arcface-r100 +POSTGRES_VERSION=1.2.0 diff --git a/custom-builds/SubCenter-ArcFace-r100/docker-compose.yml b/custom-builds/SubCenter-ArcFace-r100/docker-compose.yml index 4b5992694f..e6c3670149 100644 --- a/custom-builds/SubCenter-ArcFace-r100/docker-compose.yml +++ b/custom-builds/SubCenter-ArcFace-r100/docker-compose.yml @@ -5,7 +5,8 @@ volumes: services: compreface-postgres-db: - image: postgres:11.5 + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always container_name: "compreface-postgres-db" environment: - POSTGRES_USER=${postgres_username} @@ -16,6 +17,7 @@ services: compreface-admin: image: ${registry}compreface-admin:${ADMIN_VERSION} + restart: always container_name: "compreface-admin" environment: - POSTGRES_USER=${postgres_username} @@ -28,12 +30,15 @@ services: - EMAIL_FROM=${email_from} - EMAIL_PASSWORD=${email_password} - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B depends_on: - compreface-postgres-db - compreface-api compreface-api: image: ${registry}compreface-api:${API_VERSION} + restart: always container_name: "compreface-api" depends_on: - compreface-postgres-db @@ -44,18 +49,32 @@ services: - SPRING_PROFILES_ACTIVE=dev - API_JAVA_OPTS=${compreface_api_java_options} - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} compreface-fe: image: ${registry}compreface-fe:${FE_VERSION} + restart: always container_name: "compreface-ui" ports: - "8000:80" depends_on: - compreface-api - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms compreface-core: image: ${registry}compreface-core:${CORE_VERSION} + restart: always container_name: "compreface-core" environment: - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-2} + - UWSGI_THREADS=${uwsgi_threads:-1} + diff --git a/db/Dockerfile b/db/Dockerfile new file mode 100644 index 0000000000..cb450d902c --- /dev/null +++ b/db/Dockerfile @@ -0,0 +1,4 @@ +FROM postgres:11.5 + +COPY ./initdb.sql /docker-entrypoint-initdb.d/initdb.sql + diff --git a/db/initdb.sql b/db/initdb.sql new file mode 100644 index 0000000000..8f66f97c06 --- /dev/null +++ b/db/initdb.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + diff --git a/dev/.env b/dev/.env index f8102b7fb5..5af5fd0795 100644 --- a/dev/.env +++ b/dev/.env @@ -11,11 +11,19 @@ email_password= enable_email_server=false save_images_to_db=true compreface_api_java_options=-Xmx8g -compreface_admin_java_options=-Xmx8g +compreface_admin_java_options=-Xmx1g +max_file_size=5MB +max_request_size=10M +max_detect_size=640 +uwsgi_processes=2 +uwsgi_threads=1 +connection_timeout=10000 +read_timeout=60000 ADMIN_VERSION=latest API_VERSION=latest FE_VERSION=latest CORE_VERSION=latest +POSTGRES_VERSION=latest # ND4J library classifier values: # * linux-x86_64, windows-x86_64, macosx-x86_64 - old CPUs (pre 2012) and low power x86 (Atom, Celeron): no AVX support (usually) diff --git a/dev/Dockerfile b/dev/Dockerfile index 03189233e1..1383fffe34 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,21 +1,25 @@ -FROM maven:3.6.3-jdk-11-slim as build +FROM maven:3.8.2-eclipse-temurin-17 as build ARG ND4J_CLASSIFIER WORKDIR /workspace/compreface LABEL intermidiate_frs=true COPY pom.xml . +COPY api/pom.xml api/ +COPY admin/pom.xml admin/ +COPY common/pom.xml common/ +RUN mvn -B clean install -DskipTests -Dcheckstyle.skip -Dasciidoctor.skip -Djacoco.skip -Dmaven.gitcommitid.skip -Dspring-boot.repackage.skip -Dmaven.exec.skip=true -Dmaven.install.skip -Dmaven.resources.skip COPY api api COPY admin admin COPY common common -RUN mvn -B clean package -Dmaven.test.skip=true -Dmaven.site.skip=true -Dmaven.javadoc.skip=true -Dnd4j.classifier=$ND4J_CLASSIFIER +RUN mvn package -Dmaven.test.skip=true -Dmaven.site.skip=true -Dmaven.javadoc.skip=true -Dnd4j.classifier=$ND4J_CLASSIFIER -FROM openjdk:11.0.8-jre-slim as frs_core +FROM eclipse-temurin:17-jre-focal as frs_core ARG DIR=/workspace/compreface COPY --from=build ${DIR}/api/target/*.jar /home/app.jar ENTRYPOINT ["sh","-c","java $API_JAVA_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -jar /home/app.jar"] -FROM openjdk:11.0.8-jre-slim as frs_crud +FROM eclipse-temurin:17-jre-focal as frs_crud ARG DIR=/workspace/compreface COPY --from=build ${DIR}/admin/target/*.jar /home/app.jar ARG APPERY_API_KEY ENV APPERY_API_KEY ${APPERY_API_KEY} -ENTRYPOINT ["sh","-c","java $ADMIN_JAVA_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -jar /home/app.jar"] \ No newline at end of file +ENTRYPOINT ["sh","-c","java $ADMIN_JAVA_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -jar /home/app.jar"] diff --git a/dev/README.md b/dev/README.md index a49757e83d..db45f63695 100644 --- a/dev/README.md +++ b/dev/README.md @@ -14,7 +14,7 @@ 3. In case some containers are working, they should be stopped: `docker-compose down` 4. Clean all local datebases and images: `docker system prune --volumes` 5. Go to Dev folder `cd dev` -6. Run `sh start.sh` and make sure http://localhost:4200/ starts +6. Run `sh start.sh` and make sure http://localhost:4200/ starts (Only for UI contributors) #### How to start for UI development: diff --git a/dev/docker-compose-gpu.yml b/dev/docker-compose-gpu.yml index 4e04b5bc73..9d5e8f8e49 100644 --- a/dev/docker-compose-gpu.yml +++ b/dev/docker-compose-gpu.yml @@ -5,8 +5,9 @@ volumes: services: compreface-postgres-db: - image: postgres:11.5 + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} container_name: "compreface-postgres-db" + build: ../db ports: - "6432:5432" environment: @@ -37,6 +38,8 @@ services: - EMAIL_FROM=${email_from} - EMAIL_PASSWORD=${email_password} - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B depends_on: - compreface-postgres-db - compreface-api @@ -62,6 +65,10 @@ services: - SPRING_PROFILES_ACTIVE=dev - API_JAVA_OPTS=${compreface_api_java_options} - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} compreface-fe: image: ${registry}compreface-fe:${FE_VERSION} @@ -74,6 +81,10 @@ services: depends_on: - compreface-api - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms compreface-core: image: ${registry}compreface-core:${CORE_VERSION} @@ -91,3 +102,12 @@ services: - GPU_IDX=0 environment: - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-1} + - UWSGI_THREADS=${uwsgi_threads:-1} + healthcheck: + test: curl --fail http://localhost:3000/healthcheck || exit 1 + interval: 10s + retries: 0 + start_period: 0s + timeout: 1s diff --git a/dev/docker-compose-tpu.yml b/dev/docker-compose-tpu.yml new file mode 100644 index 0000000000..e941ebf067 --- /dev/null +++ b/dev/docker-compose-tpu.yml @@ -0,0 +1,119 @@ +version: '3.4' + +volumes: + postgres-data: + +services: + compreface-postgres-db: + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always + container_name: "compreface-postgres-db" + build: + context: ../db + ports: + - "6432:5432" + environment: + - POSTGRES_USER=${postgres_username} + - POSTGRES_PASSWORD=${postgres_password} + - POSTGRES_DB=${postgres_db} + volumes: + - postgres-data:/var/lib/postgresql/data + + compreface-admin: + image: ${registry}compreface-admin:${ADMIN_VERSION} + restart: always + build: + context: ../java + dockerfile: ../dev/Dockerfile + target: frs_crud + container_name: "compreface-admin" + ports: + - "8081:8080" + # - "5006:5005" # Uncomment this port mapping for application debug + environment: + - POSTGRES_USER=${postgres_username} + - POSTGRES_PASSWORD=${postgres_password} + - POSTGRES_URL=jdbc:postgresql://${postgres_domain}:${postgres_port}/${postgres_db} + - SPRING_PROFILES_ACTIVE=dev + - ENABLE_EMAIL_SERVER=${enable_email_server} + - EMAIL_HOST=${email_host} + - EMAIL_USERNAME=${email_username} + - EMAIL_FROM=${email_from} + - EMAIL_PASSWORD=${email_password} + - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + depends_on: + - compreface-postgres-db + - compreface-api + + compreface-api: + image: ${registry}compreface-api:${API_VERSION} + restart: always + build: + context: ../java + dockerfile: ../dev/Dockerfile + target: frs_core + args: + - ND4J_CLASSIFIER=${ND4J_CLASSIFIER} + container_name: "compreface-api" + ports: + - "8082:8080" + # - "5005:5005" # Uncomment this port mapping for application debug + depends_on: + - compreface-postgres-db + environment: + - POSTGRES_USER=${postgres_username} + - POSTGRES_PASSWORD=${postgres_password} + - POSTGRES_URL=jdbc:postgresql://${postgres_domain}:${postgres_port}/${postgres_db} + - SPRING_PROFILES_ACTIVE=dev + - API_JAVA_OPTS=${compreface_api_java_options} + - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} + + compreface-fe: + image: ${registry}compreface-fe:${FE_VERSION} + restart: always + build: + context: ../ui + dockerfile: docker-prod/Dockerfile + container_name: "compreface-ui" + ports: + - "8000:80" + depends_on: + - compreface-api + - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms + + compreface-core: + image: ${registry}compreface-core:${CORE_VERSION} + restart: always + container_name: "compreface-core" + ports: + - "3300:3000" + build: + context: ../embedding-calculator + args: + - FACE_DETECTION_PLUGIN=facenet.coralmtcnn.FaceDetector + - CALCULATION_PLUGIN=facenet.coralmtcnn.Calculator + - EXTRA_PLUGINS=facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector,facenet.PoseEstimator + environment: + - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-2} + - UWSGI_THREADS=${uwsgi_threads:-1} + healthcheck: + test: curl --fail http://localhost:3000/healthcheck || exit 1 + interval: 10s + retries: 0 + start_period: 0s + timeout: 1s + privileged: true + devices: + - /dev/bus/usb \ No newline at end of file diff --git a/dev/docker-compose.dev.ui.yml b/dev/docker-compose.dev.ui.yml index 3f682b25f6..0ec802e6d5 100644 --- a/dev/docker-compose.dev.ui.yml +++ b/dev/docker-compose.dev.ui.yml @@ -5,3 +5,7 @@ services: build: context: ../ui dockerfile: docker-dev/Dockerfile + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms diff --git a/dev/docker-compose.env.yml b/dev/docker-compose.env.yml new file mode 100644 index 0000000000..ebb88bbf18 --- /dev/null +++ b/dev/docker-compose.env.yml @@ -0,0 +1,24 @@ +################################################################################################# +# Docker compose override file for deploy allication stack to the CompreFace project environments +# +# Usage: HOSTNAME=$HOSTNAME sudo docker-compose -f docker-compose.yml -f docker-compose.env.yml up -d +# +# Note: a) We need to provide $HOSTNAME to mount TLS certs correctly for each environment. +# b) We also mount nginx configuration with HTTPS instead of the default one. +# +################################################################################################# +version: '3.4' + +volumes: + postgres-data: + +services: + compreface-fe: + volumes: + - ./nginx/templates:/etc/nginx/templates:ro + - /etc/letsencrypt:/etc/letsencrypt:ro + environment: + - NGINX_HOST=$HOSTNAME + ports: + - "8000:80" + - "443:443" diff --git a/dev/docker-compose.multiplatform.yml b/dev/docker-compose.multiplatform.yml new file mode 100644 index 0000000000..a4b08a43c6 --- /dev/null +++ b/dev/docker-compose.multiplatform.yml @@ -0,0 +1,32 @@ +version: '3.4' + +services: + compreface-postgres-db: + build: + platforms: + - "linux/amd64" + - "linux/arm64" + + compreface-admin: + build: + platforms: + - "linux/amd64" + - "linux/arm64" + + compreface-api: + build: + platforms: + - "linux/amd64" + - "linux/arm64" + + compreface-fe: + build: + platforms: + - "linux/amd64" + - "linux/arm64" + + compreface-core: + build: + platforms: + - "linux/amd64" + - "linux/arm64" diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index d96cd13ab4..3e8b047851 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -5,8 +5,11 @@ volumes: services: compreface-postgres-db: - image: postgres:11.5 + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always container_name: "compreface-postgres-db" + build: + context: ../db ports: - "6432:5432" environment: @@ -18,6 +21,7 @@ services: compreface-admin: image: ${registry}compreface-admin:${ADMIN_VERSION} + restart: always build: context: ../java dockerfile: ../dev/Dockerfile @@ -25,7 +29,7 @@ services: container_name: "compreface-admin" ports: - "8081:8080" - - "5006:5005" + # - "5006:5005" # Uncomment this port mapping for application debug environment: - POSTGRES_USER=${postgres_username} - POSTGRES_PASSWORD=${postgres_password} @@ -37,12 +41,15 @@ services: - EMAIL_FROM=${email_from} - EMAIL_PASSWORD=${email_password} - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B depends_on: - compreface-postgres-db - compreface-api compreface-api: image: ${registry}compreface-api:${API_VERSION} + restart: always build: context: ../java dockerfile: ../dev/Dockerfile @@ -52,7 +59,7 @@ services: container_name: "compreface-api" ports: - "8082:8080" - - "5005:5005" + # - "5005:5005" # Uncomment this port mapping for application debug depends_on: - compreface-postgres-db environment: @@ -62,9 +69,14 @@ services: - SPRING_PROFILES_ACTIVE=dev - API_JAVA_OPTS=${compreface_api_java_options} - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} compreface-fe: image: ${registry}compreface-fe:${FE_VERSION} + restart: always build: context: ../ui dockerfile: docker-prod/Dockerfile @@ -74,9 +86,14 @@ services: depends_on: - compreface-api - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms compreface-core: image: ${registry}compreface-core:${CORE_VERSION} + restart: always container_name: "compreface-core" ports: - "3300:3000" @@ -84,3 +101,12 @@ services: context: ../embedding-calculator environment: - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-2} + - UWSGI_THREADS=${uwsgi_threads:-1} + healthcheck: + test: curl --fail http://localhost:3000/healthcheck || exit 1 + interval: 10s + retries: 0 + start_period: 0s + timeout: 1s diff --git a/dev/nginx/templates/nginx.conf.template b/dev/nginx/templates/nginx.conf.template new file mode 100644 index 0000000000..1e767b83fa --- /dev/null +++ b/dev/nginx/templates/nginx.conf.template @@ -0,0 +1,80 @@ +upstream frsadmin { + server compreface-admin:8080 fail_timeout=10s max_fails=5; +} + +upstream frsapi { + server compreface-api:8080 fail_timeout=10s max_fails=5; +} + +upstream frscore { + server compreface-core:3000 fail_timeout=10s max_fails=5; +} + +server { + listen 80; + server_name ui; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name ui; + ssl_certificate /etc/letsencrypt/live/$NGINX_HOST/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/$NGINX_HOST/privkey.pem; + + client_max_body_size 10M; + + location / { + root /usr/share/nginx/html/; + index index.html; + try_files $uri $uri/ /index.html =404; + } + + location /admin/ { + proxy_pass http://frsadmin/admin/; + } + + location /api/v1/ { + + proxy_read_timeout 60000ms; + proxy_connect_timeout 10000ms; + + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key' always; + + proxy_pass http://frsapi/api/v1/; + } + + location /core/ { + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key' always; + + proxy_pass http://frscore/; + } + + location ~ ^/(api|admin)/(swagger-ui.html|webjars|swagger-resources|v2/api-docs)(.*) { + proxy_set_header 'Host' $http_host; + proxy_pass http://frs$1/$2$3$is_args$args; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 4b5992694f..ac07b0bc7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,8 @@ volumes: services: compreface-postgres-db: - image: postgres:11.5 + image: ${registry}compreface-postgres-db:${POSTGRES_VERSION} + restart: always container_name: "compreface-postgres-db" environment: - POSTGRES_USER=${postgres_username} @@ -15,6 +16,7 @@ services: - postgres-data:/var/lib/postgresql/data compreface-admin: + restart: always image: ${registry}compreface-admin:${ADMIN_VERSION} container_name: "compreface-admin" environment: @@ -28,11 +30,14 @@ services: - EMAIL_FROM=${email_from} - EMAIL_PASSWORD=${email_password} - ADMIN_JAVA_OPTS=${compreface_admin_java_options} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B depends_on: - compreface-postgres-db - compreface-api compreface-api: + restart: always image: ${registry}compreface-api:${API_VERSION} container_name: "compreface-api" depends_on: @@ -44,8 +49,13 @@ services: - SPRING_PROFILES_ACTIVE=dev - API_JAVA_OPTS=${compreface_api_java_options} - SAVE_IMAGES_TO_DB=${save_images_to_db} + - MAX_FILE_SIZE=${max_file_size} + - MAX_REQUEST_SIZE=${max_request_size}B + - CONNECTION_TIMEOUT=${connection_timeout:-10000} + - READ_TIMEOUT=${read_timeout:-60000} compreface-fe: + restart: always image: ${registry}compreface-fe:${FE_VERSION} container_name: "compreface-ui" ports: @@ -53,9 +63,23 @@ services: depends_on: - compreface-api - compreface-admin + environment: + - CLIENT_MAX_BODY_SIZE=${max_request_size} + - PROXY_READ_TIMEOUT=${read_timeout:-60000}ms + - PROXY_CONNECT_TIMEOUT=${connection_timeout:-10000}ms compreface-core: + restart: always image: ${registry}compreface-core:${CORE_VERSION} container_name: "compreface-core" environment: - ML_PORT=3000 + - IMG_LENGTH_LIMIT=${max_detect_size} + - UWSGI_PROCESSES=${uwsgi_processes:-2} + - UWSGI_THREADS=${uwsgi_threads:-1} + healthcheck: + test: curl --fail http://localhost:3000/healthcheck || exit 1 + interval: 10s + retries: 0 + start_period: 0s + timeout: 1s diff --git a/docs/Architecture-and-scalability.md b/docs/Architecture-and-scalability.md index 8588352890..59f1e61f6e 100644 --- a/docs/Architecture-and-scalability.md +++ b/docs/Architecture-and-scalability.md @@ -1,48 +1,88 @@ # Architecture and Scalability -By default, CompreFace is delivered as docker-compose file, so you can easily start it with one command. However, CompreFace could be scaled up to distribute computations on different servers and achieve high availability. -This section describes the architecture of CompreFace, each of its components, and suggestions on how to scale the system. +CompreFace is delivered as a docker-compose file by default, so you can +easily start it with one command. However, CompreFace could be scaled up +to distribute computations on different servers and achieve high +availability. This section describes the architecture of CompreFace, +each of its components, and suggestions on how to scale the system. ## CompreFace architecture diagram ![CompreFace architecture diagram](https://user-images.githubusercontent.com/3736126/107855144-5db83580-6e29-11eb-993a-46cdc0c82812.png) ## Balancer + UI -Container name in docker-compose file: compreface-fe + +Container name in the docker-compose file: `compreface-fe` This container runs Nginx that serves CompreFace UI. -In the default config, it’s also used as the main gateway - Nginx proxies user requests to admin and api servers. +In the default config, it's also used as the main gateway - Nginx +proxies user requests to admin and API servers. ## Admin server -Container name in docker-compose file: compreface-admin -Admin server is a Spring Boot application and it’s responsible for all operations that are done on UI. Admin server connects to PostgreSQL database to store the data. +Container name in the docker-compose file: `compreface-admin` + +Admin server is a Spring Boot application, and it's responsible for all +operations that are done on UI. Admin server connects to PostgreSQL +database to store the data. + +## API servers + +Container name in the docker-compose file: `compreface-api` -## Api servers -Container name in docker-compose file: compreface-api +API servers handle all user API calls: face recognition, face detection, +and face verification. -Api servers handle all user API calls: face recognition, face detection, and face verification. +It provides API key validation, proxies images to Embedding servers, and +classifies the face. For face classification, we use the ND4J library. -It provides API key validation, proxies images to Embedding servers, and classifies the face. For face classification, we use the ND4J library. +By default, the number of API servers in the config is 1, but for production +environments to increase possible bandwidth and achieve high +availability, there should be at least two such servers, and they should +be on different machines. In addition, the data synchronization is +implemented via PostgreSQL notifications, so if, for example, you add a +new face to a collection, all other servers know about it and can +recognize this new face. -In the default config number of API servers is 1, but for production environments to increase possible bandwidth and to achieve high availability, there should be at least two such servers and they should be on different machines. The data synchronization is implemented via postgreSQL notifications, so if for example, you add a new face to a collection, all other servers will know about it and will be able to recognize this new face. +Classification is not a very heavy operation as embedding calculation +and doesn't require GPU in most cases. API server connects to PostgreSQL +database to store the data. -Classification is not a very heavy operation as embedding calculation and in most cases doesn’t require GPU. API server connects to PostgreSQL database to store the data. +There is a `PYTHON_UR`L environment variable that tells this container where +to send requests to `compreface-core` containers. +Default value: http://compreface-core:3000. + +There is a `PYTHON_URL` environment variable that tells this container where to send requests to `compreface-core` containers. Default value: `http://compreface-core:3000`. ## Embedding Servers -Container name in docker-compose file: compreface-core -Embedding server is responsible for running neural networks. It calculates embeddings from faces and makes all plugin recognitions like age and gender detection. These servers are stateless, they don't have a connection to a database and they don't require any synchronization between each other. +Container name in the docker-compose file: `compreface-core` + +The embedding server is responsible for running neural networks. It +calculates embeddings from faces and makes all plugin recognitions like +age and gender detection. These servers are stateless, don't have a +connection to a database, and don't require any synchronization between +them. -In the default config number of API servers is 1, but for production environments to increase possible bandwidth and to achieve high availability, there should be at least two such servers and they should be on different machines. +By default, the number of embedding servers in the config is 1, but for production +environments to increase possible bandwidth and achieve high +availability, there should be at least two such servers, and they should +be on different machines. -Running neural networks is a very heavy operation. The total performance of the system highly depends on these nodes. This is why we recommend using highly performant nodes to run Embedding Servers, ideally with GPU support. To learn more about how to run CompreFace with GPU, see custom builds documentation. +Running neural networks is a very heavy operation. Therefore, the total +performance of the system highly depends on these nodes. That is why we +recommend using highly performant nodes to run Embedding Servers, +ideally with GPU support. To learn more about how to run CompreFace with +GPU, see [custom-builds documentation](Custom-builds.md). ## PostgreSQL + Default docker-compose configuration includes postgreSQL database. -If you want CompreFace to connect to your database, you need to provide such environment variables for compreface-admin and compreface-api containers: +If you want CompreFace to connect to your database, you need to provide +such environment variables for `compreface-admin` and `compreface-api` +containers: * POSTGRES_PASSWORD * POSTGRES_URL * POSTGRES_USER diff --git a/docs/Configuration.md b/docs/Configuration.md index 9fe274c136..9885038b95 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -1,22 +1,48 @@ # Configuration -In the [release](https://github.com/exadel-inc/CompreFace/releases) archive and all custom builds there is a `.env` file with -configuration options for CompreFace. For production systems -we recommend looking through them and set up CompreFace accordingly +In the [release](https://github.com/exadel-inc/CompreFace/releases) +archive and all custom builds, there is a `.env` file with configuration +options for CompreFace. For production systems, we recommend looking +through them and set up CompreFace accordingly -* `registry` - this is the docker hub registry. For release and pre-build images, it should be set to `exadel/` value -* `postgres_password` - password for Postgres database. It should be changed for production systems from the default value. -* `postgres_domain` - the domain where Postgres database is run -* `postgres_port` - Postgres database port -* `enable_email_server` - if true, it will enable email verification for users. You should set email_host, email_username, and email_password variables for the correct work. -* `email_host` - a host of the email provider. It should be set if `enable_email_server` variable is true -* `email_username` - a username of email provider for authentication. It should be set if `enable_email_server` variable is true -* `email_password` - a password of email provider for authentication. It should be set if `enable_email_server` variable is true -* `email_from` - this value will see users in `from` fields when they receive emails from CompreFace. Corresponds to `From` field in rfc2822. Optional, if not set, then `email_username` will be used instead -* `save_images_to_db` - should the CompreFace save photos to the database. Be careful, [migrations](Face-data-migration.md) could be run only if this value is `true` -* `compreface_api_java_options` - java options of compreface-api container -* `compreface_admin_java_options` - java options of compreface-admin container -* `ADMIN_VERSION` - docker image tag of compreface-admin container -* `API_VERSION` - docker image tag of compreface-api container -* `FE_VERSION` - docker image tag of compreface-fe container -* `CORE_VERSION` - docker image tag of compreface-core container +- `registry` - this is the docker hub registry. For release and + pre-build images, it should be set to `exadel/` value +- `postgres_username` - username for Postgres database +- `postgres_password` - password for Postgres database. It should be + changed for production systems from the default value. +- `postgres_db` - name for Postgres database +- `postgres_domain` - the domain where Postgres database is run +- `postgres_port` - Postgres database port +- `enable_email_server` - if true, it enables email verification for + users. You should set email_host, email_username, and email_password + variables for the correct work. +- `email_host` - a host of the email provider. It should be set if the + `enable_email_server` variable is true +- `email_username` - a username of email provider for authentication. + It should be set if the `enable_email_server` variable is true +- `email_password` - a password of the email provider for + authentication. It should be set if the `enable_email_server` + variable is true +- `email_from` - this value reads users in the `from` fields when + they receive emails from CompreFace. Corresponds to `From` field in + rfc2822. Optional, if not set, then `email_username` is used instead +- `save_images_to_db` - should the CompreFace save photos to the + database. Be careful, [migrations](Face-data-migration.md) could be + run only if this value is `true`. Doesn't work in 0.6.0 and 0.6.1 version, please use 0.5.1 version or >0.6.1 version instead +- `compreface_api_java_options` - java options of compreface-api + container +- `compreface_admin_java_options` - java options of compreface-admin + container +- `max_file_size` - maximum image size acceptable to CompreFace. It must be less than or equal to `max_request_size` +- `max_request_size` - maximum request size for a multipart/form-data acceptable to CompreFace. It must be greater than or equal to + `max_file_size` +- `uwsgi_processes` - the number of uWSGI processes +- `uwsgi_threads` - the number of uWSGI threads +- `connection_timeout` - request connection timeout. It is used to set the connection timeout for the Nginx proxy and the Feign client +- `read_timeout` - request read timeout. It is used to set the read timeout for the Nginx proxy and the Feign client +- `ADMIN_VERSION` - docker image tag of the compreface-admin container +- `API_VERSION` - docker image tag of the compreface-api container +- `FE_VERSION` - docker image tag of the compreface-fe container +- `CORE_VERSION` - docker image tag of the compreface-core container +- `POSTGRES_VERSION` - docker image tag of the compreface-postgres-db container +- `max_detect_size` - if the width or height of the input image exceeds this value, it will be resized to fit within this size diff --git a/docs/Custom-builds.md b/docs/Custom-builds.md index 70bbcaecf1..aeb877cdd2 100644 --- a/docs/Custom-builds.md +++ b/docs/Custom-builds.md @@ -1,77 +1,94 @@ # Custom Builds -There is always a trade off between the face recognition accuracy, max throughput of the system and even hardware support. +There is always a trade-off between the face recognition accuracy, the +system's max throughput, and even hardware support. -By default, CompreFace release contains configuration that could be run on the widest variety of hardware. +By default, the CompreFace release contains configuration that could be +run on the widest variety of hardware. -The downside of this build is that it's not optimized for the latest generations of CPU and doesn't support GPU. +The downside of this build is that it's not optimized for the latest +generations of CPU and doesn't support GPU. -With custom-builds we aim to cover as many cases as we can. They are not tested as good as the default build, and we encourage community +With custom-builds, we aim to cover as many cases as we can. They are +not tested as well as the default build, and we encourage the community to report any bugs related to these builds. ## List of custom-builds -You can find the list of custom-builds [here](../custom-builds/README.md) - +You can find the list of custom builds [here](../custom-builds/README.md) ## Contribution -We also encourage community to share their own builds, we will add them to our list with notice that this is community build. -## How to choose a build +We also encourage the community to share their builds; we will add them +to our list with notice that this is a community build. -Different builds are fit for different purposes - some of them have higher accuracy, but the performance on CPU is low, others optimized -for low-performance hardware and have acceptable accuracy. You have to make your own choice in this trade off. But generally you can -follow this rules: +## How to choose a build -- If you want to run real-time face recognition, we recommend choosing builds with GPU support. -- If you need to run face recognition on old or low performance systems, we recommend to use builds with models originally created for - mobile -- Do not take blindly the most accurate model. The accuracy is not so different between models, but the required hardware resources - could differ dramatically +Different builds are fit for different purposes - some of them have +higher accuracy, but the performance on CPU is low; others are optimized +for low-performance hardware and have acceptable accuracy. Of course, +you have to make your own choice in this trade-off. But generally, you +can follow these rules: + +- If you want to run real-time face recognition, we recommend choosing + builds with GPU support. +- If you need to run face recognition on old or low-performance + systems, we recommend using builds with models initially created for + mobile +- Do not take the most accurate model blindly. The accuracy does not + vary significantly between models, but the required hardware + resources could differ dramatically ## How to run custom-builds -Running custom-build is very similar to running the default build - all you need to do is to open the corresponding folder and run +Running custom-build is very similar to running the default build - all +you need to do is open the corresponding folder and run `docker-compose up -d`. Things to consider: -- If you run CompreFace from the custom-build folder, it will create a new docker volume, - so you won't see your saved information. To run custom-build with your previously saved information, - you need to copy files from custom-build to folder with the original build (and replace the original files) -- In most cases, face recognition models are not interchangeable, - this means that all you saved examples from the old build won't work on new builds. - See [migrations documentation](Face-data-migration.md) to know what is the options. -- Do not run two instances of CompreFace simultaneously without changing the port. - To change the port go to `docker-compose` file and change the post for `compreface-fe` container. - - + +- If you run CompreFace from the custom-build + folder, it creates a new docker volume so that you won't see your saved + information. To run custom-build with your previously saved information, + you need to copy files from custom-build to folder with the original + build (and replace the original files) +- In most cases, face recognition + models are not interchangeable; this means that all you saved examples + from the old build won't work on new builds. See [migrations + documentation](Face-data-migration.md) to know what is the options. +- Do not run two instances of CompreFace simultaneously without changing the + port. To change the port, go to the `docker-compose` file and change the + post for the `compreface-fe` container. + ## How to build your own custom-build ### Custom models -CompreFace supports two face recognition libraries - FaceNet and InsightFace. It means CompreFace can run any model that can run this -libraries. All you need to do is -1. Upload your model to Google Drive and add it to one the following files into the `Calculator` class: -- /embedding-calculator/src/services/facescan/plugins/facenet/facenet.py -- /embedding-calculator/src/services/facescan/plugins/insightface/insightface.py -2. Take the `docker-compose` file from `/dev` folder as a template -3. Specify new model name in build arguments. For more information look at [this documentation](https://github. - com/exadel-inc/CompreFace/tree/master/embedding-calculator#run-service). E.g. here is a part of `docker-compose` file for building with custom model with GPU support: -``` +CompreFace supports two face recognition libraries - FaceNet and +InsightFace. It means CompreFace can run any model that can run these +libraries. So all you need to do is +1. Upload your model to Google Drive and add it to one of the following files into the `Calculator` class: + - /embedding-calculator/src/services/facescan/plugins/facenet/facenet.py + - /embedding-calculator/src/services/facescan/plugins/insightface/insightface.py +2. Take the `docker-compose` file from `/dev` folder as a template +3. Specify new model names in build arguments. For more information, look + at [this documentation](https://github.com/exadel-inc/CompreFace/tree/master/embedding-calculator#run-service). E.g. here is a part of the + `docker-compose` file for building with a custom model with GPU support: +```yaml compreface-core: - image: ${registry}compreface-core:${CORE_VERSION} - container_name: "compreface-core" - ports: - - "3300:3000" - runtime: nvidia - build: - context: ../embedding-calculator - args: - - FACE_DETECTION_PLUGIN=insightface.FaceDetector@retinaface_r50_v1 - - CALCULATION_PLUGIN=insightface.Calculator@arcface_r100_v1 - - EXTRA_PLUGINS=insightface.LandmarksDetector,insightface.GenderDetector,insightface.AgeDetector - - BASE_IMAGE=${registry}compreface-core-base:base-cuda100-py37 - - GPU_IDX=0 - environment: - - ML_PORT=3000 -``` + image: ${registry}compreface-core:${CORE_VERSION} + container_name: "compreface-core" + ports: + - "3300:3000" + runtime: nvidia + build: + context: ../embedding-calculator + args: + - FACE_DETECTION_PLUGIN=insightface.FaceDetector@retinaface_r50_v1 + - CALCULATION_PLUGIN=insightface.Calculator@arcface_r100_v1 + - EXTRA_PLUGINS=insightface.LandmarksDetector,insightface.GenderDetector,insightface.AgeDetector,insightface.facemask.MaskDetector,insightface.PoseEstimator + - BASE_IMAGE=compreface-core-base:base-cuda100-py37 + - GPU_IDX=0 + environment: + - ML_PORT=3000 +``` \ No newline at end of file diff --git a/docs/Face-Recognition-Similarity-Threshold.md b/docs/Face-Recognition-Similarity-Threshold.md index 23348e1f25..507c175f04 100644 --- a/docs/Face-Recognition-Similarity-Threshold.md +++ b/docs/Face-Recognition-Similarity-Threshold.md @@ -1,13 +1,23 @@ # Face Recognition Similarity Threshold -The result of CompreFace face recognition and face verification services is a similarity between faces. Even if you upload faces of two different people, you will still receive the result, but the similarity will be low. The user must determine for himself whether this is the same person or not using similarity. -The level of similarity the user accepts, as big enough, we call similarity threshold. +The result of CompreFace face recognition and face verification services +is a similarity between faces. Even if you upload the faces of two +different people, you still receive the result, but the similarity is +low. Therefore, the user must determine for himself whether this is the +same person or not using similarity. The level of similarity the user +accepts, as big enough, we call similarity threshold. ## How to choose the face similarity threshold -No Face Recognition Service has 100% accuracy, so there always will be errors in recognition. -If a user chooses too low threshold, then some unknown faces will be recognized as known. -If a user chooses too high threshold, then some known faces will be recognized as unknown. -CompreFace calculates similarity in a way, so most correct guesses will be with a threshold of more than 0.5, and the most incorrect guesses will be with a threshold of less than 0.5. Still, we recommend for high-security systems set the threshold more than 0.5. -This is the distribution of similarities for a custom dataset of 50,000 faces for FaceNet model (blue - is incorrect guesses, red is correct): -distribution of similarities \ No newline at end of file +No Face Recognition Service has 100% accuracy, so there always appear +errors in recognition. If a user chooses too low a threshold, then some +unknown faces are recognized as known. If a user chooses too high a +threshold, then some known faces are recognized as unknown. CompreFace +calculates similarity so that most correct guesses have a threshold of +more than 0.5, and the most incorrect guesses have a threshold of less +than 0.5. Still, we recommend for high-security systems set the +threshold more than 0.5. This is the distribution of similarities for a +custom dataset of 50,000 faces for the FaceNet model (blue - is +incorrect guesses, red is correct): + +distribution of similarities diff --git a/docs/Face-data-migration.md b/docs/Face-data-migration.md index 4e46af245f..a887f68160 100644 --- a/docs/Face-data-migration.md +++ b/docs/Face-data-migration.md @@ -2,36 +2,43 @@ ## When do you need to migrate data -When you upload a new known image to the Face Collection, -CompreFace uses a neural network model to calculate an embedding (also known as face features), -which is basically an array of 512 or 128 numbers. -Then CompreFace saves it to the database to use in the future comparison - when you upload a face to recognize, -CompreFace again calculates an embedding and compares it to saved embeddings. - -The important thing here is that every neural network model will calculate different embeddings. -It means that it makes sense to compare embeddings calculated by the same neural network model. - -CompreFace doesn't change the neural network model during its work, so normally you don’t need to migrate your face data. -If you want to try [custom build](Custom-builds.md), be very careful - look at the table [here](../custom-builds/README.md), -column `Face recognition model` - if the model changed, you need to run a migration. +When you upload a new known image to the Face Collection, CompreFace +uses a neural network model to calculate an embedding (also known as +face features), which is an array of 512 or 128 numbers. Then CompreFace +saves it to the database to use in the future comparison - when you +upload a face to recognize, CompreFace again calculates an embedding and +compares it to saved embeddings. + +The important thing here is that every neural network model calculates +different embeddings. Therefore, it means that it makes sense to compare +embeddings calculated by the same neural network model. + +CompreFace doesn't change the neural network model during its work, so +you usually don't need to migrate your face data. If you want to try +[custom build](Custom-builds.md), be very careful - look at the table +[here](../custom-builds/README.md), column `Face recognition model` - if +the model changed, you need to run a migration. ## Limitations -If you run CompreFace in the [“not saving images to database” mode](Configuration.md)(`save_images_to_db=false`), -you won’t be able to migrate data as the original images are required for migration. +If you run CompreFace in the ["not saving images to database" +mode](Configuration.md)(`save_images_to_db=false`), you won't be +able to migrate data as the original images are required for migration. -The only solution here is to delete all images from Face Collection and upload them again. +The only solution here is to delete all images from Face Collection and +upload them again. ## How to perform a migration -Current migration was written for internal usage and wasn’t tested enough, so please do a backup copy of the database and perform migration at your own risk. +Current migration was written for internal usage and wasn't tested +enough, so please do a backup copy of the database and perform migration +at your own risk. REST request to start migration: -```shell -curl -i -X POST \ -'http://localhost:8000/api/v1/migrate' -``` + curl -i -X POST \ + 'http://localhost:8000/api/v1/migrate' -This rest endpoint is asynchronous, it will start the migration and return a response immediately. -To understand if the migration is successful, please look at logs for “Migration successfully finished” text. +This rest endpoint is asynchronous; it starts the migration and returns +a response immediately. Please look at logs for "Migration successfully +finished" text to understand if the migration is successful. diff --git a/docs/Face-services-and-plugins.md b/docs/Face-services-and-plugins.md index edf84b9d47..0169b87773 100644 --- a/docs/Face-services-and-plugins.md +++ b/docs/Face-services-and-plugins.md @@ -8,41 +8,133 @@ CompreFace supports these face services and plugins: * Gender detection plugin * Landmarks detection plugin * Calculator plugin +* Face mask detection plugin +* Head pose plugin -## Services +# Services -To use face service you need to create it in an application on UI. The type of service depends on your application needs. Each service has its own REST API context and there is no possibility to change the service type after creation. Here is a short description of each of them: -* Face recognition service is used for face identification. This means that you first need to upload known faces to faces collection and then recognize unknown faces among them. When you upload an unknown face, the service returns the most similar faces to it. Also, face recognition service supports verify endpoint to check if this person from face collection is the correct one. The possible cases include: +To use face service you need to create it in an application on UI. +The type of service depends on your application needs. +Each service has its own REST API context and there is no possibility to change the service type after creation. +Here is a short description of each of them: + +## Face detection + +Face detection service is used to detect all faces in the image. +It doesn’t recognize faces, just finds them on the image. + +**Cases of use** + +The most useful cases include face plugins for face analysis: + * gather statistics on how your store popular among different genders + * gather statistics on among what ages your event is popular + * get landmark information to know where customers look at + * gather statistics on how many customers in the store + * recognize if all customers wear masks properly + +**How to test** + +1. On the CompreFace application page, at the bottom of the frame, click Create button. +2. In the Create Service dialog, from the Type drop-down menu, select DETECTION. +3. Enter the name of the service you are going to create. +4. From the list of the services in the Services frame, select the service you created; you can use search field to filter the services. +5. Click Test button near in the row of the service you want to launch. +6. On the service page, open or drag-and-drop the picture to analyze. +7. The service will display the original picture with marks near every face. + +**Output** + +Below the picture, you can see the Request processed, and the Response to the request. +The Response is the output which CompreFace provides via [API](Rest-API-description.md#face-detection-service). + +Example: + +![Example](https://user-images.githubusercontent.com/3736126/146967067-c6413d3e-3b23-45ad-abe8-0f8bc8f4800f.png) + +## Face recognition + +Face recognition service is used for face identification. This means that you first need to upload known faces to faces collection and +then recognize unknown faces among them. When you upload an unknown face, the service returns the most similar faces to it. +Also, face recognition service supports verify endpoint to check if this person from face collection is the correct one. + +**Cases of use** + +The possible cases include: * when you have photos of employees and want to recognize strangers in the office * when you have photos of conference attendees and want to track who was interested in which topics. * when you have photos of VIP guests and you want to find them among the crowd very quickly. -* Face verification service is used to check if this person is the correct one. The service compares two faces you send to the rest endpoint and returns their similarity. The possible cases include: + +**How to test** + +1. On the CompreFace application page, at the bottom of the frame, click Create button. +2. In the Create Service dialog, from the Type drop-down menu, select RECOGNITION. +3. Enter the name of the service you are going to create. +4. From the list of the services in the Services frame, select the service you created; you can use search field to filter the services. +5. Click Test button near in the row of the service you want to launch. +6. On the service page, open or drag-and-drop the picture to analyze. +7. The service will display the original picture with marks near every face. + +**Output** + +Below the picture, you can see the Request processed, and the Response to the request. +The Response is the output which CompreFace provides via [API](Rest-API-description.md#face-recognition-service). + +Example: + +![image](https://user-images.githubusercontent.com/3736126/146967594-40684d12-e106-43b2-92ad-6a34176ddf87.png) + +## Face verification + +Face verification service is used to check if this person is the correct one. +The service compares two faces you send to the rest endpoint and returns their similarity. + +**Cases of use** + +The possible cases include: * when a customer provides you an ID or driving license and you need to verify if this is him * when a user connects his social network account to your application and you want to verify if this is him -* Face detection service is used to detect all faces in the image. It doesn’t recognize faces, just find them on the image. The most useful cases include face plugins for face analysis: - * gather statistics on how your store popular among different genders - * gather statistics on among what ages your event is popular - * get landmark information to know where customers look at - * gather statistics on how many customers in the store -## Face plugins +**How to test** + +1. On the CompreFace application page, at the bottom of the frame, click Create button. +2. In the Create Service dialog, from the Type drop-down menu, select VERIFICATION. +3. Enter the name of the service you are going to create. +4. From the list of the services in the Services frame, select the service you created; you can use search field to filter the services. +5. Click Test button near in the row of the service you want to launch. +6. On the service page, open or drag-and-drop two pictures to compare their content. +7. The service will display the original picture with marks near every face. + +**Output** + +Below the picture, you can see the Request processed, and the Response to the request. +The Response is the output which CompreFace provides via [API](Rest-API-description.md#face-verification-service). + +Example: + +![image](https://user-images.githubusercontent.com/3736126/146967889-ba8bdd9b-359f-4970-bfe0-71f3e6d21692.png) + +# Face plugins Face plugins could be used with any of the face services. By default, face services return only bounding boxes and similarity if applicable. To add more information in response you can add face plugins in your request. To add a plugin you need to list comma-separated needed plugins in the query `face_plugins` parameter. This parameter is supported by all face recognition services. Example: + ```shell -curl -X POST "http://localhost:8000/api/v1/recognition/recognize?face_plugins=age,gender,landmarks" \ +curl -X POST "http://localhost:8000/api/v1/recognition/recognize?face_plugins=age,gender,landmarks,mask,pose" \ -H "Content-Type: multipart/form-data" \ -H "x-api-key: " \ -F file= ``` -This request will recognize faces on the image and return additional information about age, gender, and landmarks. + +This request will recognize faces on the image and return additional information about age, gender, head pose, face mask, and landmarks. The list of possible plugins: * age - returns the supposed range of a person’s age in format [min, max] * gender - returns the supposed person’s gender * landmarks - returns face landmarks. This plugin is supported by all configurations and returns 5 points of eyes, nose, and mouth * calculator - returns face embeddings. +* pose - returns head pose in format: `{"pitch": 0.0,"roll": 0.0,"yaw": 0.0}` +* mask - returns if the person wears a mask. Possible results: `without_mask`, `mask_worn_incorrectly`, `mask_worn_correctly`. Learn more about [mask plugin](Mask-detection-plugin.md) * landmarks2d106 - returns face landmarks. This plugin is supported only by the configuration that uses insightface library. It’s not - available by default. More information about landmarks [here](https://github.com/deepinsight/insightface/tree/master/alignment/coordinateReg#visualization). + available by default. More information about landmarks [here](https://github.com/deepinsight/insightface/tree/ce3600a74209808017deaf73c036759b96a44ccb/alignment/coordinate_reg#visualization). diff --git a/docs/Gathering-anonymous-statistics.md b/docs/Gathering-anonymous-statistics.md index f43b66d579..46b1e0ecd3 100644 --- a/docs/Gathering-anonymous-statistics.md +++ b/docs/Gathering-anonymous-statistics.md @@ -1,26 +1,52 @@ -# Gathering Anonymous Statistics +# Gathering Anonymous Statistics -To better understand which features we should add to the service and how we can improve it further we implemented functionality for gathering anonymous statistics. This section aims to describe what exact information we collect. +To better understand which features we should add to the service and +improve it further, we implemented functionality for gathering anonymous +statistics. This section aims to describe what exact information we +collect. -We respect the privacy of our users, this is why all statistics are strictly anonymized before sent to our servers. There is no possibility to de-anonymize received information. In short, we collect information about how many users, applications, services, and faces your installation has. During the first user sign up there is a sign “Agree to send anonymous statistics”. By checking it you agree with Exadel Privacy Policy and agree to send anonymous statistics to our servers. +We respect the privacy of our users; this is why all statistics are +strictly anonymized before being sent to our servers. There is no +possibility to de-anonymize received information. In short, we collect +information about how many users, applications, services, and faces your +installation has. During the first user sign-up, there is a sign "Agree +to send anonymous statistics." By checking it, you agree with Exadel +Privacy Policy and agree to send anonymous statistics to our servers. #### What we collect: -* Event of user registration - we record only the fact of the creation of a new user. We do not gather any information about the user -(like name, email password, etc.). -* Event of application creation - we record only the fact of the creation of a new application. We do not gather any information about the application(like name, which users have access to it, etc.). -* Event of service creation - we record only the fact of the creation of a new service and its type. We do not gather any information about - the service(like name, etc.). -* Number of saved faces in Face Recognition service Collection. Every day we record how many faces are saved in Collection in ranges: 1-10, 11-50, 51-200, 201-500, 501-2000, 2001-10000, 10001-50000, 50001-200000, 200001-1000000, 1000001+. We do not gather any information about the faces(like face name, embedding, etc.). + +- Event of user registration - we record only the fact of the creation + of a new user. We do not gather any information about the user (like + name, email password, etc.). +- Event of application creation - we record only the fact of the + creation of a new application. We do not gather any information + about the application(like name, which users have access to it, + etc.). +- Event of service creation - we record only the creation of a new + service and its type. We do not gather any information about the + service(like name, etc.). +- The number of saved faces in Face Recognition service Collection. + Every day we record how many faces are saved in Collection in + ranges: 1-10, 11-50, 51-200, 201-500, 501-2000, 2001-10000, + 10001-50000, 50001-200000, 200001-1000000, 1000001+. We do not + gather any information about the faces(like face name, embedding, + etc.). #### What we do NOT collect: -* Any personal information of CompreFace users or the end-users -* Any names you use in CompreFace -* Any information about hardware, software, or location of the host machine -During the first start, we assign to the CompreFace installation the `install_guid` variable. This variable is totally random, there is no possibility to retrieve any information from it, the only purpose of this variable is to understand that gathered statistics were sent from one machine. We send it in every request to our server to understand that this is the same installation as before. +- Any personal information of CompreFace users or the end-users +- Any names you use in CompreFace +- Any information about hardware, software, or location of the host + machine -#### Examples of saved data: +During the first start, we assign to the CompreFace installation the +`install_guid` variable. This variable is random; there is no +possibility to retrieve any information from it; the only purpose of +this variable is to understand that gathered statistics were sent from +one machine. We send it in every request to our server to understand +that this is the same installation as before. +#### Examples of saved data: ```csv "_createdAt:date","install_guid:string","action_name:string" "2021-03-15 09:16:31.676","560eee90-5fca-11eb-988b-0242ac120003","USER_CREATE" @@ -29,7 +55,6 @@ During the first start, we assign to the CompreFace installation the `install_gu "2021-03-15 09:16:32.607","560eee90-5fca-11eb-988b-0242ac120003","FACE_VERIFICATION_CREATE" "2021-03-15 09:16:32.998","560eee90-5fca-11eb-988b-0242ac120003","FACE_RECOGNITION_CREATE" ``` - ```csv "_createdAt:date","install_guid:string","collection_guid:string","faces_range:string" "2021-03-13 13:25:49.700","59638de4-5fca-11eb-848b-0242ac120002","a3d5dda8-b53a-4465-a44e-f1c3c81c7551","501-2000" @@ -37,11 +62,16 @@ During the first start, we assign to the CompreFace installation the `install_gu "2021-03-13 13:25:50.003","59638de4-5fca-11eb-848b-0242ac120002","39c1925d-a1a9-4d44-8eb3-6acf132b89f2","1-10" "2021-03-13 13:25:50.763","59638de4-5fca-11eb-848b-0242ac120002","794dd0ec-ac88-4552-90a8-f0bb0ddcee1e","201-500" ``` - #### How we use the data -The data is used to understand the popularity of different services, how many faces usually are saved to face collection and how many users use CompreFace an ongoing basis. -We do not provide this data to third parties in any case. -We still can publish aggregated data in self-promotional goals, like "CompreFace has N active users" or "CompreFace is successfully used with face collections that stores more than 1 million faces". +The data is used to understand the popularity of different services, how +many faces usually are saved to face collection and how many users use +CompreFace on an ongoing basis. We do not provide this data to third +parties in any case. However, we still can publish aggregated data in +self-promotional goals, like "CompreFace has N active users" or +"CompreFace is successfully used with face collections that stores more +than 1 million faces". -If you have any questions about the privacy policy, what data we collect or how we use it, please [contact us](mailto:compreface.support@exadel.com) \ No newline at end of file +If you have any questions about the privacy policy, what data we +collect, or how we use it, please [get in touch with +us](mailto:compreface.support@exadel.com) diff --git a/docs/How-to-Use-CompreFace.md b/docs/How-to-Use-CompreFace.md index ea93993497..aa72d4e8a4 100644 --- a/docs/How-to-Use-CompreFace.md +++ b/docs/How-to-Use-CompreFace.md @@ -1,127 +1,149 @@ # How to Use CompreFace -**Step 1.** Install and run CompreFace using our [Getting Started guide](../README.md#getting-started-with-compreface) - -**Step 2.** You need to sign up for the system and log in into the account you’ve just created or use the one you already have. After that, the system redirects you to the main page. - -**Step 3.** Create an application (left section) using the "Create" link at the bottom of the page. An application is where you can create and manage your Face Collections. - -**Step 4.** Enter your application by clicking on its name. Here you will have two options: you can either add new users and manage -their access roles or create new [Face Services](Face-services-and-plugins.md). - -**Step 5.** To recognize subjects among the known subjects, you need to create Face Recognition Service. After creating a new Face -Service, you will see it in the Services List with an appropriate name and API key. After this step, you can look at our [demos](#demos). - -**Step 6.** To add known subjects to your Face Collection of Face Recognition Service, you can use REST API. -Once you’ve [uploaded all known faces](Rest-API-description.md#add-an-example-of-a-subject), -you can test the collection using [REST API](Rest-API-description.md#recognize-faces-from-a-given-image) or the TEST page. -We recommend that you use an image size no higher than 5MB, as it could slow down the request process. The supported image formats include JPEG/PNG/JPG/ICO/BMP/GIF/TIF/TIFF. - -**Step 7.** Upload your photo and let our open-source face recognition system match the image against the Face Collection. If you use a -UI for face recognition, you will see the original picture with marks near every face. If you use [REST API](Rest-API-description.md#recognize-faces-from-a-given-image), you will receive a response in JSON format. - -JSON contains an array of objects that represent each recognized face. Each object has the following fields: - -1. `subject` - person identificator -2. `similarity` - gives the confidence that this is the found subject -3. `probability` - gives the confidence that this is a face -4. `x_min`, `x_max`, `y_min`, `y_max` are coordinates of the face in the image +**Step 1.** Install and run CompreFace using our [Getting Started +guide](../README.md#getting-started-with-compreface) + +**Step 2.** You need to sign up for the system and login into the +account you've just created or use the one you already have. After +that, the system redirects you to the main page. + +**Step 3.** Create an application (left section) using the "Create" +link at the bottom of the page. An application is where you can create +and manage your Face Collections. + +**Step 4.** Enter your application by clicking on its name. Here, you +have two options: adding new users and managing their access roles or +creating new [Face Services](Face-services-and-plugins.md). + +**Step 5.** To recognize subjects among the known subjects, you need to +create a Face Recognition Service. After creating a new Face Service, +you can see it in the Services List with an appropriate name and API +key. After this step, you can look at our [demos](#demos). + +**Step 6.** To add known subjects to your Face Collection of Face +Recognition Service, you can use REST API. Once you've [uploaded all +known faces](Rest-API-description.md#add-an-example-of-a-subject), you +can test the collection using [REST +API](Rest-API-description.md#recognize-faces-from-a-given-image) or the +TEST page. We recommend using an image size no higher than 5MB, as it +could slow down the request process. The supported image formats include +JPEG/PNG/JPG/ICO/BMP/GIF/TIF/TIFF. + +**Step 7.** Upload your photo and let our open-source face recognition +system match the image against the Face Collection. Using a UI for face +recognition, you can see the original picture with marks near every +face. Using [REST +API](Rest-API-description.md#recognize-faces-from-a-given-image), you +receive a response in JSON format. + +JSON contains an array of objects that represent each recognized face. +Each object has the following fields: + +1. `subject` - person identifier +2. `similarity` - gives a confidence that this is the found subject +3. `probability` - gives the confidence that this is a face +4. `x_min`, `x_max`, `y_min`, `y_max` are coordinates of the face in + the image +``` -```json -{ - "result": [ { - "box": { - "probability": 0.99583, - "x_max": 551, - "y_max": 364, - "x_min": 319, - "y_min": 55 - }, - "subjects": [ + "result": [ { - "similarity": 0.99593, - "subject": "lisan" + "box": { + "probability": 0.99583, + "x_max": 551, + "y_max": 364, + "x_min": 319, + "y_min": 55 + }, + "subjects": [ + { + "similarity": 0.99593, + "subject": "lisan" + } + ] + }, + { + } ] - }, - { - } - ] -} ``` ## Demos -1. [tutorial_demo.html](./demos/tutorial_demo.html) +1. [tutorial_demo.html](./demos/tutorial_demo.html) -This demo shows the most simple example of Face recognition service usage. -To run demo, just open html file in browser. -API key for this demo was created on **step 5** of [How to Use CompreFace](#how-to-use-compreface) +This demo shows the most simple example of Face recognition service +usage. To run a demo, open an HTML file in a browser. API key for this +demo was created on **step 5** of [How to Use +CompreFace](#how-to-use-compreface) -2. [webcam_demo.html](./demos/webcam_demo.html) +2. [webcam_demo.html](./demos/webcam_demo.html) -This demo shows the most simple webcam demo for Face recognition service. -To run demo, just open html file in browser. -API key for this demo was created on **step 5** of [How to Use CompreFace](#how-to-use-compreface) +This demo shows the most simple webcam demo for Face recognition +service. To run a demo, open an HTML file in a browser. API key for this +demo was created on **step 5** of [How to Use +CompreFace](#how-to-use-compreface) ## Code Snippets -Here is a JavaScript code snippet that loads a new image to your Face Collection: +Here is a JavaScript code snippet that loads a new image to your Face +Collection: ```js -function saveNewImageToFaceCollection(elem) { - let subject = encodeURIComponent(document.getElementById("subject").value); - let apiKey = document.getElementById("apiKey").value; - let formData = new FormData(); - let photo = elem.files[0]; - - formData.append("file", photo); - - fetch('http://localhost:8000/api/v1/recognition/faces/?subject=' + subject, - { - method: "POST", - headers: { - "x-api-key": apiKey - }, - body: formData - } - ).then(r => r.json()).then( - function (data) { - console.log('New example was saved', data); - }) - .catch(function (error) { - alert('Request failed: ' + JSON.stringify(error)); - }); -} + function saveNewImageToFaceCollection(elem) { + let subject = encodeURIComponent(document.getElementById("subject").value); + let apiKey = document.getElementById("apiKey").value; + let formData = new FormData(); + let photo = elem.files[0]; + + formData.append("file", photo); + + fetch('http://localhost:8000/api/v1/recognition/faces/?subject=' + subject, + { + method: "POST", + headers: { + "x-api-key": apiKey + }, + body: formData + } + ).then(r => r.json()).then( + function (data) { + console.log('New example was saved', data); + }) + .catch(function (error) { + alert('Request failed: ' + JSON.stringify(error)); + }); + } ``` -This function sends the image to our server and shows results in a text area: +This function sends the image to our server and shows results in a text +area: ```js -function recognizeFace(elem) { - let apiKey = document.getElementById("apiKey").value; - let formData = new FormData(); - let photo = elem.files[0]; - - formData.append("file", photo); - - fetch('http://localhost:8000/api/v1/recognition/recognize', - { - method: "POST", - headers: { - "x-api-key": apiKey - }, - body: formData - } - ).then(r => r.json()).then( - function (data) { - document.getElementById("result").innerHTML = JSON.stringify(data); - }) - .catch(function (error) { - alert('Request failed: ' + JSON.stringify(error)); - }); -} -``` + function recognizeFace(elem) { + let apiKey = document.getElementById("apiKey").value; + let formData = new FormData(); + let photo = elem.files[0]; + + formData.append("file", photo); + + fetch('http://localhost:8000/api/v1/recognition/recognize', + { + method: "POST", + headers: { + "x-api-key": apiKey + }, + body: formData + } + ).then(r => r.json()).then( + function (data) { + document.getElementById("result").innerHTML = JSON.stringify(data); + }) + .catch(function (error) { + alert('Request failed: ' + JSON.stringify(error)); + }); + } +``` \ No newline at end of file diff --git a/docs/Installation-options.md b/docs/Installation-options.md new file mode 100644 index 0000000000..5230c5d734 --- /dev/null +++ b/docs/Installation-options.md @@ -0,0 +1,125 @@ +# Installation (Deployment) options + +Exadel CompreFace consists of several services and a database. +Full architecture description and scaling tips you can find [here](Architecture-and-scalability.md). +Each service is put to docker image for simpler usage, and they can be run separately. +However, for a better user experience, CompreFace provides three distribution options that help install CompreFace easier. +By default, CompreFace is delivered as a docker-compose configuration. But there are more options to install and run CompreFace. +Each of them has its benefits and disadvantages. + +| Distribution | Advantages | Disadvantages | Best for | +|:------------------------:|:------------------------------------------------------------:|:--------------------------------------------------:|:-----------------------------------------------------:| +| Docker Compose (Default) | Simple configuration
Simple run
Passed QA regression | Requires Docker Compose
Runs on single machine | Local installation | +| Kubernetes | Simple to scale | Requires Kubernetes cluster | Production installation | +| Single docker container | Simple configuration
Simple run | Least reliable option
Runs on single machine | Local installation if Docker Compose is not supported | + +## Docker Compose + +Docker-compose configuration allows simply run, configure, stop and restart CompreFace. +To install CompreFace using docker-compose just follow instructions in [getting started](../README.md#getting-started-with-compreface) + +### Maintaining tips + +1. After you run CompreFace, wait at least 30 seconds until it starts. + Do not stop it during this time, as it may corrupt database data during data migration. +2. You can run `docker-compose ps` to see all CompreFace services. + There should be 5 CompreFace services: compreface-core, compreface-api, compreface-admin, compreface-ui, compreface-postgres-db. + If at least one of the services is not in `Up` status - CompreFace failed to start. +3. To see the logs of service, run `docker-compose logs -f `, e.g. `docker-compose logs -f compreface-api`. + You also can run `docker-compose logs -f` to see the logs of all CompreFace services. +4. Docker-compose automatically restarts all services if they fail. It also automatically starts them after you restart your machine. +5. If you want to stop CompreFace, run `docker-compose stop`. + You can also stop each container one by one, e.g. `docker-compose stop compreface-core`. +6. To start stopped Compreface, run `docker-compose start`. + You can also start each container one by one, e.g. `docker-compose start compreface-core`. +7. If you want to restart CompreFace, run `docker-compose restart`. + You can also restart each container one by one, e.g. `docker-compose restart compreface-core`. +8. All the data is stored locally on your machine. It is stored in a named docker volume. + This guarantees that if you stop or delete CompreFace docker containers, you won’t lose the data. + To find the volume name, run `docker volume ls`, the name should be `_postgres-data`, e.g. `compreface_061_postgres-data`. +9. If you want to clear CompreFace installation, first stop it with `docker-compose stop`. + Then delete the volume, e.g. `docker volume rm compreface_061_postgres-data`. Then run CompreFace again `docker-compose up -d`. +10. To update the CompreFace version or change custom build, download new `docker-compose.yml` and `.env` files. + Stop CompreFace with `docker-compose down`. Copy new files into the old CompreFace folder. Then run CompreFace with `docker-compose up -d`. + +### Troubleshooting + +1. Problem: `compreface-core` doesn’t run. + + Probable solution: please check if you have supported CPU or GPU. The default version of CompreFace requires an x86 processor and AVX support. + +2. Problem: `compreface-admin` doesn’t start and there are logs like `Waiting for changelog lock....` + + Solution: clear CompreFace installation (see #Maintaining-tips) + +## Kubernetes + +You can find all Kubernetes scripts in CompreFace [Kubernetes repository](https://github.com/exadel-inc/compreface-kubernetes). + +## Single docker container + +Except for other distribution options, here all services and the database are placed in one docker image. +The obvious advantage of this approach is that it is the simplest way to start CompreFace. +However, it has some limitations in maintaining and troubleshooting. +E.g. it’s very difficult to stop or restart services one by one. +[Supervisord](http://supervisord.org/) was used to maintain several services in one Docker container. + +**Requirements:** + +1. Docker Engine for Linux or Docker Desktop for Windows and macOS +2. CompreFace could be run on most modern computers with x86 processor and AVX support. + To check AVX support on Linux run `lscpu | grep avx` command + +To install CompreFace single docker container run command (you don’t need anything to download manually): + +```commandline +docker run -d --name=CompreFace -v compreface-db:/var/lib/postgresql/data -p 8000:80 exadel/compreface +``` + +To use your own database for storing the data, specify these environment variables: POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_URL, EXTERNAL_DB, e.g.: + +```commandline +docker run -d --name=CompreFace -e "POSTGRES_URL=jdbc:postgresql://url:port/db_name" -e POSTGRES_USER=user -e POSTGRES_PASSWORD=pass -e EXTERNAL_DB=true -p 8000:80 exadel/compreface +``` + +To run the custom version of CompreFace, specify it in the end, e.g.: +```commandline +docker run -d --name=CompreFace -v compreface-db:/var/lib/postgresql/data -p 8000:80 exadel/compreface:0.6.0 +``` + +To run custom builds you can use corresponding tags, e.g.: +```commandline +docker run -d --name=CompreFace -v compreface-db:/var/lib/postgresql/data -p 8000:80 exadel/compreface:1.0.0-mobilenet +``` + +To run version with GPU, you need to specify `--runtime=nvidia` and corresponding tag, e.g.: +```commandline +docker run -d --name=CompreFace -v compreface-db:/var/lib/postgresql/data --runtime=nvidia -p 8000:80 exadel/compreface:1.0.0-arcface-r100-gpu +``` + +### Maintaining tips + +1. Start CompreFace in a single docker container takes at least 45 seconds. + So long start is because of manual timings that help to start services in the right order. +2. There is a possibility that the database starts too slow, then service `compreface-admin` will fail. + Supervisord will restart it automatically and CompreFace should start properly. +3. To check if the run is finished, you can check the logs `docker logs CompreFace -f`. + If you see `exited: startup (exit status 0; expected)` log, it is finished. +4. To check if CompreFace is run, run `docker ps`. It should be a container with the name `CompreFace`. + You set the name of the container in run command `name=CompreFace` +5. `compreface-db` in the run command is the name of the volume, all your data is stored locally in this volume. + This guarantees that if you stop or delete CompreFace docker containers, you won’t lose the data. +6. You can use environment variables from `docker-compose` version, e.g. to set API server limit you can run: +```commandline +docker run -d -e "API_JAVA_OPTS=-Xmx8g" --name=CompreFace -v compreface-db:/var/lib/postgresql/data -p 8000:80 exadel/compreface` +``` +7. By default, docker won’t restart CompreFace if it fails or after your restart your machine. + You can add this by adding `--restart=always` in run command: +```commandline +docker run -d --name=CompreFace -v compreface-db:/var/lib/postgresql/data -p 8000:80 --restart=always exadel/compreface +``` +8. If you want to stop CompreFace, run `docker stop CompreFace`. +9. To start stopped Compreface, run `docker start CompreFace`. +10. If you want to restart CompreFace, run `docker restart CompreFace`. +11. If you want to clear CompreFace installation, first stop it with `docker stop CompreFace`. Remove container with `docker rm CompreFace`. Then delete the volume `docker volume rm compreface-db`. Then run CompreFace again. +12. To update the CompreFace version or change custom build, stop CompreFace with `docker stop CompreFace`. Remove container with `docker rm CompreFace`. Then run the new CompreFace version. diff --git a/docs/Mask-detection-plugin.md b/docs/Mask-detection-plugin.md new file mode 100644 index 0000000000..7c3f3476ab --- /dev/null +++ b/docs/Mask-detection-plugin.md @@ -0,0 +1,46 @@ +# Face mask detection plugin + +A Mask detection plugin can be used to detect if the person wears a mask +correctly automatically. There are three possible results: +`without_mask`, `mask_worn_incorrectly`, `mask_worn_correctly`. + +There was no suitable free and ready-to-use model for face mask +detection at the moment of adding this plugin, so we created our model. + + Disclaimer: + Software developers, not medical experts, created the plugin. + The plugin doesn't contain the recommendations of how to use and wear face mask correctly. + The accuracy of the model is not 100%. + Please use the plugin at your own risk. + +# Face mask detection example + +![results](https://user-images.githubusercontent.com/3736126/130656086-3167421e-f697-4837-8cf9-e3889d49a44d.png) + +# Training process + +## Dataset + +We used four publicly available datasets for training the model: + +1. [Kaggle face mask detection + dataset](https://www.kaggle.com/andrewmvd/face-mask-detection) +2. [Kaggle medical masks dataset images + tfrecords](https://www.kaggle.com/ivandanilovich/medical-masks-dataset-images-tfrecords) +3. [Kaggle face mask detection dataset + \#2](https://www.kaggle.com/wobotintelligence/face-mask-detection-dataset?select=train.csv) +4. [MAFA + dataset](https://drive.google.com/drive/folders/1nbtM1n0--iZ3VVbNGhocxbnBGhMau_OG) + +We extracted faces with masks from the first dataset (around 4k images), +faces without a mask from the first two datasets (around 4k images), +faces with masks worn incorrectly from all four datasets (around 2k +images). Then we duplicated each incorrect worn mask image with data +augmentation (see augmentation.py) to achieve class balance. + +## Train + +InceptionV3 was cut off on a mixed 7 layer to improve speed and was used +as a backbone. The final model with 97.2 % accuracy is used by default +and can be found +[here](https://drive.google.com/file/d/1jm2Wd2JEZxhS8O1JjV-kfBOyOYUMxKHq/view?usp=sharing) diff --git a/docs/README.md b/docs/README.md index 664c63355e..7e662122f1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,5 +9,8 @@ * [Configuration](Configuration.md) * [Architecture and Scalability](Architecture-and-scalability.md) * [Custom Builds](Custom-builds.md) +* [Face data migration](Face-data-migration.md) * [User Roles System](User-Roles-System.md) -* [Gathering Anonymous Statistics](Gathering-anonymous-statistics.md) \ No newline at end of file +* [Face Mask Detection Plugin](Mask-detection-plugin.md) +* [Gathering Anonymous Statistics](Gathering-anonymous-statistics.md) +* [Installation Options](/docs/Installation-options.md) \ No newline at end of file diff --git a/docs/Rest-API-description.md b/docs/Rest-API-description.md index a501dfddfc..1aefe28377 100644 --- a/docs/Rest-API-description.md +++ b/docs/Rest-API-description.md @@ -2,27 +2,52 @@ ## Table Of Contents -+ [Face Recognition Service Endpoints](#face-recognition-service-endpoints) - + [Add a Subject](#add-a-subject) - + [Rename a Subject](#rename-a-subject) - + [Delete a Subject](#delete-a-subject) - + [Delete All Subjects](#delete-all-subjects) - + [List Subjects](#list-subjects) ++ [Postman REST API documentation](#postman-documentation) ++ [Face Recognition Service](#face-recognition-service) + + [Managing Subjects](#managing-subjects) + + [Add a Subject](#add-a-subject) + + [Rename a Subject](#rename-a-subject) + + [Delete a Subject](#delete-a-subject) + + [Delete All Subjects](#delete-all-subjects) + + [List Subjects](#list-subjects) + + [Managing Subject Examples](#managing-subject-examples) + [Add an Example of a Subject](#add-an-example-of-a-subject) - + [Recognize Faces from a Given Image](#recognize-faces-from-a-given-image) + [List of All Saved Examples of the Subject](#list-of-all-saved-examples-of-the-subject) + [Delete All Examples of the Subject by Name](#delete-all-examples-of-the-subject-by-name) + [Delete an Example of the Subject by ID](#delete-an-example-of-the-subject-by-id) + + [Delete Multiple Examples](#delete-multiple-examples) + [Direct Download an Image example of the Subject by ID](#direct-download-an-image-example-of-the-subject-by-id) + [Download an Image example of the Subject by ID](#download-an-image-example-of-the-subject-by-id) - + [Verify Faces from a Given Image](#verify-faces-from-a-given-image) + + [Recognize Faces from a Given Image](#recognize-faces-from-a-given-image) + + [Verify Faces from a Given Image](#verify-faces-from-a-given-image) + [Face Detection Service](#face-detection-service) + [Face Verification Service](#face-verification-service) + [Base64 Support](#base64-support) ++ [Recognition and verification using embedding](#recognition-and-verification-using-embedding) To know more about face services and face plugins visit [this page](Face-services-and-plugins.md). -## Face Recognition Service Endpoints +## Postman documentation + +There is a [Postman REST API documentation](https://documenter.getpostman.com/view/17578263/UUxzAnde) +that covers the same REST endpoint. Postman documentation supports snippets on the most popular programming languages. + +## Face Recognition Service + +### Managing Subjects + +These endpoints allow you to work with subjects. + +The most popular case of subject usage is to assign a subject to one person. +So, to upload several images of one person, you need to upload them to one subject. +As a result, when you perform face recognition, you find a person who is on the image. + +Another case of subject usage is assigning a photo of several people as a subject. +In this case, you need to detect all faces on the image and then save them to one subject. +As a result, when you perform face recognition, you find all photos on which there is the person who is on the image. +You don’t need to work with subjects explicitly. +You can just upload a new example of the subject and the subject will be created automatically. +Or if you delete all the examples of the subject, it will be deleted automatically. ### Add a Subject ```since 0.6 version``` @@ -34,13 +59,14 @@ you can [upload an example](#add-an-example-of-a-subject) without an existing su curl -X POST "http://localhost:8000/api/v1/recognition/subjects" \ -H "Content-Type: application/json" \ -H "x-api-key: " \ --d '{"subject: "}' +-d '{"subject": ""}' ``` -| Element | Description | Type | Required | Notes | -| ------------------- | ----------- | ------ | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | application/json | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| subject | body param | string | required | is the name of the subject. It can be a person name, but it can be any string | + +| Element | Description | Type | Required | Notes | +|--------------|-------------|--------|----------|-------------------------------------------------------------------------------| +| Content-Type | header | string | required | application/json | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| subject | body param | string | required | is the name of the subject. It can be a person name, but it can be any string | Response body on success: ```json @@ -49,9 +75,9 @@ Response body on success: } ``` -| Element | Type | Description | -| -------- | ------ | -------------------------- | -| subject | string | is the name of the subject | +| Element | Type | Description | +|---------|--------|----------------------------| +| subject | string | is the name of the subject | ### Rename a Subject ```since 0.6 version``` @@ -65,11 +91,11 @@ curl -X PUT "http://localhost:8000/api/v1/recognition/subjects/" \ -H "x-api-key: " \ -d '{"subject: "}' ``` -| Element | Description | Type | Required | Notes | -| ------------------- | ----------- | ------ | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | application/json | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| subject | body param | string | required | is the name of the subject. It can be a person name, but it can be any string | +| Element | Description | Type | Required | Notes | +|--------------|-------------|--------|----------|-------------------------------------------------------------------------------| +| Content-Type | header | string | required | application/json | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| subject | body param | string | required | is the name of the subject. It can be a person name, but it can be any string | Response body on success: ```json @@ -78,9 +104,9 @@ Response body on success: } ``` -| Element | Type | Description | -| -------- | ------- | ----------------- | -| updated | boolean | failed or success | +| Element | Type | Description | +|---------|---------|-------------------| +| updated | boolean | failed or success | ### Delete a Subject ```since 0.6 version``` @@ -92,11 +118,11 @@ curl -X DELETE "http://localhost:8000/api/v1/recognition/subjects/" \ -H "Content-Type: application/json" \ -H "x-api-key: " ``` -| Element | Description | Type | Required | Notes | -| ------------------- | ----------- | ------ | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | application/json | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| subject | body param | string | required | is the name of the subject. It can be a person name, but it can be any string | +| Element | Description | Type | Required | Notes | +|--------------|-------------|--------|----------|-------------------------------------------------------------------------------| +| Content-Type | header | string | required | application/json | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| subject | body param | string | required | is the name of the subject. It can be a person name, but it can be any string | Response body on success: ```json @@ -105,9 +131,9 @@ Response body on success: } ``` -| Element | Type | Description | -| -------- | ------ | -------------------------- | -| subject | string | is the name of the subject | +| Element | Type | Description | +|---------|--------|----------------------------| +| subject | string | is the name of the subject | ### Delete All Subjects ```since 0.6 version``` @@ -119,10 +145,10 @@ curl -X DELETE "http://localhost:8000/api/v1/recognition/subjects" \ -H "Content-Type: application/json" \ -H "x-api-key: " ``` -| Element | Description | Type | Required | Notes | -| ------------------- | ----------- | ------ | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | application/json | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| Element | Description | Type | Required | Notes | +|--------------|-------------|--------|----------|--------------------------------------------------------------| +| Content-Type | header | string | required | application/json | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | Response body on success: ```json @@ -131,9 +157,9 @@ Response body on success: } ``` -| Element | Type | Description | -| -------- | ------- | -------------------------- | -| deleted | integer | number of deleted subjects | +| Element | Type | Description | +|---------|---------|----------------------------| +| deleted | integer | number of deleted subjects | ### List Subjects ```since 0.6 version``` @@ -145,10 +171,10 @@ curl -X GET "http://localhost:8000/api/v1/recognition/subjects/" \ -H "Content-Type: application/json" \ -H "x-api-key: " ``` -| Element | Description | Type | Required | Notes | -| ------------------- | ----------- | ------ | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | application/json | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| Element | Description | Type | Required | Notes | +|--------------|-------------|--------|----------|--------------------------------------------------------------| +| Content-Type | header | string | required | application/json | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | Response body on success: ```json @@ -160,28 +186,37 @@ Response body on success: } ``` -| Element | Type | Description | -| -------- | ------ | -------------------------- | -| subjects | array | the list of subjects in Face Collection | +| Element | Type | Description | +|----------|-------|-----------------------------------------| +| subjects | array | the list of subjects in Face Collection | + +### Managing Subject Examples + +The subject example is basically an image of a known face that you want to save to face collection. + +When you save a subject example, CompreFace calculates the embedding of the face (faceprint) and saves it into the database. +By default, the image itself is also saved, it is needed for managing images, e.g. [download of the image](#direct-download-an-image-example-of-the-subject-by-id). You can change it using `save_images_to_db` parameter in [configuration](Configuration.md). + +One subject example is enough for face recognition, the accuracy will be high enough. But if you add more examples, the accuracy may be even better. ### Add an Example of a Subject This creates an example of the subject by saving images. You can add as many images as you want to train the system. Image should contain only one face. -```http request +```shell curl -X POST "http://localhost:8000/api/v1/recognition/faces?subject=&det_prob_threshold=" \ -H "Content-Type: multipart/form-data" \ -H "x-api-key: " \ -F file=@ ``` -| Element | Description | Type | Required | Notes | -| ------------------- | ----------- | ------ | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | multipart/form-data | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| subject | param | string | required | is the name you assign to the image you save | -| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0. | -| file | body | image | required | allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | +| Element | Description | Type | Required | Notes | +|--------------------|-------------|--------|----------|------------------------------------------------------------------------------------------------------| +| Content-Type | header | string | required | multipart/form-data | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| subject | param | string | required | is the name you assign to the image you save | +| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0. | +| file | body | image | required | allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | Response body on success: ```json @@ -192,97 +227,26 @@ Response body on success: ``` | Element | Type | Description | -| -------- | ------ | -------------------------- | +|----------|--------|----------------------------| | image_id | UUID | UUID of uploaded image | | subject | string | Subject of the saved image | -### Recognize Faces from a Given Image - -To recognize faces from the uploaded image: - -```http request -curl -X POST "http://localhost:8000/api/v1/recognition/recognize?limit=&prediction_count=&det_prob_threshold=&face_plugins=&status=" \ --H "Content-Type: multipart/form-data" \ --H "x-api-key: " \ --F file= -``` - -| Element | Description | Type | Required | Notes | -| ------------------ | ----------- | ------- | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | multipart/form-data | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| file | body | image | required | allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | -| limit | param | integer | optional | maximum number of faces on the image to be recognized. It recognizes the biggest faces first. Value of 0 represents no limit. Default value: 0 | -| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0. | -| prediction_count | param | integer | optional | maximum number of subject predictions per face. It returns the most similar subjects. Default value: 1 | -| face_plugins | param | string | optional | comma-separated slugs of face plugins. If empty, no additional information is returned. [Learn more](Face-services-and-plugins.md) | -| status | param | boolean | optional | if true includes system information like execution_time and plugin_version fields. Default value is false | - -Response body on success: -```json -{ - "result" : [ { - "age" : [ 25, 32 ], - "gender" : "female", - "embedding" : [ 9.424854069948196E-4, "...", -0.011415496468544006 ], - "box" : { - "probability" : 1.0, - "x_max" : 1420, - "y_max" : 1368, - "x_min" : 548, - "y_min" : 295 - }, - "landmarks" : [ [ 814, 713 ], [ 1104, 829 ], [ 832, 937 ], [ 704, 1030 ], [ 1017, 1133 ] ], - "subjects" : [ { - "similarity" : 0.97858, - "subject" : "subject1" - } ], - "execution_time" : { - "age" : 28.0, - "gender" : 26.0, - "detector" : 117.0, - "calculator" : 45.0 - } - } ], - "plugins_versions" : { - "age" : "agegender.AgeDetector", - "gender" : "agegender.GenderDetector", - "detector" : "facenet.FaceDetector", - "calculator" : "facenet.Calculator" - } -} -``` - -| Element | Type | Description | -| ------------------------------ | ------- | ------------------------------------------------------------ | -| age | array | detected age range. Return only if [age plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| gender | string | detected gender. Return only if [gender plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| embedding | array | face embeddings. Return only if [calculator plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| box | object | list of parameters of the bounding box for this face | -| probability | float | probability that a found face is actually a face | -| x_max, y_max, x_min, y_min | integer | coordinates of the frame containing the face | -| landmarks | array | list of the coordinates of the frame containing the face-landmarks. Return only if [landmarks plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| subjects | list | list of similar subjects with size of order by similarity | -| similarity | float | similarity that on that image predicted person | -| subject | string | name of the subject in Face Collection | -| execution_time | object | execution time of all plugins | -| plugins_versions | object | contains information about plugin versions | - ### List of All Saved Examples of the Subject To retrieve a list of subjects saved in a Face Collection: -```http request -curl -X GET "http://localhost:8000/api/v1/recognition/faces?page=&size=" \ +```shell +curl -X GET "http://localhost:8000/api/v1/recognition/faces?page=&size=&subject=" \ -H "x-api-key: " \ ``` -| Element | Description | Type | Required | Notes | -| --------- | ----------- | ------- | -------- | ----------------------------------------- | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| page | param | integer | optional | page number of examples to return. Can be used for pagination. Default value is 0. Since 0.6 version | -| size | param | integer | optional | faces on page (page size). Can be used for pagination. Default value is 20. Since 0.6 version | +| Element | Description | Type | Required | Notes | +|-----------|-------------|---------|----------|------------------------------------------------------------------------------------------------------------| +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| page | param | integer | optional | page number of examples to return. Can be used for pagination. Default value is 0. Since 0.6 version | +| size | param | integer | optional | faces on page (page size). Can be used for pagination. Default value is 20. Since 0.6 version | +| subject | param | string | optional | what subject examples endpoint should return. If empty, return examples for all subjects. Since 1.0 version| Response body on success: @@ -302,28 +266,28 @@ Response body on success: } ``` -| Element | Type | Description | -| -------------- | ------- | ------------------------------------------------------------ | -| face.image_id | UUID | UUID of the face | -| fase.subject | string | of the person, whose picture was saved for this api key | -| page_number | integer | page number | -| page_size | integer | **requested** page size | -| total_pages | integer | total pages | -| total_elements | integer | total faces | +| Element | Type | Description | +|----------------|---------|-------------------------------------------------------------------| +| face.image_id | UUID | UUID of the face | +| faсe.subject | string | of the person, whose picture was saved for this api key | +| page_number | integer | page number | +| page_size | integer | **requested** page size | +| total_pages | integer | total pages | +| total_elements | integer | total faces | ### Delete All Examples of the Subject by Name To delete all image examples of the : -```http request +```shell curl -X DELETE "http://localhost:8000/api/v1/recognition/faces?subject=" \ -H "x-api-key: " ``` -| Element | Description | Type | Required | Notes | -| --------- | ----------- | ------ | -------- | ------------------------------------------------------------ | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| Element | Description | Type | Required | Notes | +|-----------|-------------|--------|----------|------------------------------------------------------------------------------------------------| +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | | subject | param | string | optional | is the name subject. If this parameter is absent, all faces in Face Collection will be removed | Response body on success: @@ -333,9 +297,9 @@ Response body on success: } ``` -| Element | Type | Description | -| -------- | ------- | ------------------------ | -| count | integer | Number of deleted faces | +| Element | Type | Description | +|---------|---------|-------------------------| +| deleted | integer | Number of deleted faces | @@ -343,15 +307,15 @@ Response body on success: Endpoint to delete an image by ID. If no image found by id - 404. -```http request +```shell curl -X DELETE "http://localhost:8000/api/v1/recognition/faces/" \ -H "x-api-key: " ``` -| Element | Description | Type | Required | Notes | -| --------- | ----------- | ------ | -------- | ----------------------------------------- | +| Element | Description | Type | Required | Notes | +|-----------|-------------|--------|----------|--------------------------------------------------------------| | x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| image_id | variable | UUID | required | UUID of the removing face | +| image_id | variable | UUID | required | UUID of the removing face | Response body on success: ``` @@ -361,25 +325,59 @@ Response body on success: } ``` -| Element | Type | Description | -| -------- | ------ | ------------------------------------------------------------ | -| image_id | UUID | UUID of the removed face | +| Element | Type | Description | +|----------|--------|-------------------------------------------------------------------| +| image_id | UUID | UUID of the removed face | | subject | string | of the person, whose picture was saved for this api key | + +### Delete Multiple Examples + ```since 1.0 version``` + +To delete several subject examples: + ```shell +curl -X POST "http://localhost:8000/api/v1/recognition/faces/delete" \ +-H "Content-Type: application/json" \ +-H "x-api-key: " \ +-d '["","", ..., ""]' +``` + +| Element | Description | Type | Required | Notes | +|-----------------|-------------|--------|----------|--------------------------------------------------------------| +| service_api_key | header | string | required | api key of the Face recognition service, created by the user | +| image_id | variable | UUID | required | UUID of the removing face | + + + +Response body on success: +``` +{ + "image_id": , + "subject": +} +``` + +| Element | Description | Type | +|-----------------|-----------------------------------------------------------|--------| +| image_id | UUID of the removed face | UUID | +| subject | of the person, whose picture was saved for this api key | string | + +If some image ids are not exists, they will be ignored + ### Direct Download an Image example of the Subject by ID ```since 0.6 version``` You can paste this URL into the html tag to show the image. -```http request +```shell curl -X GET "http://localhost:8000/api/v1/static//images/" ``` -| Element | Description | Type | Required | Notes | -| --------------- | ----------- | ------ | -------- | ----------------------------------------- | +| Element | Description | Type | Required | Notes | +|-----------------|-------------|--------|----------|--------------------------------------------------------------| | service_api_key | variable | string | required | api key of the Face recognition service, created by the user | -| image_id | variable | UUID | required | UUID of the image to download | +| image_id | variable | UUID | required | UUID of the image to download | Response body is binary image. Empty bytes if image not found. @@ -389,48 +387,148 @@ Response body is binary image. Empty bytes if image not found. To download an image example of the Subject by ID: -```http request +```shell curl -X GET "http://localhost:8000/api/v1/recognition/faces//img" -H "x-api-key: " ``` -| Element | Description | Type | Required | Notes | -| --------- | ----------- | ------ | -------- | ----------------------------------------- | +| Element | Description | Type | Required | Notes | +|-----------|-------------|--------|----------|--------------------------------------------------------------| | x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| image_id | variable | UUID | required | UUID of the image to download | +| image_id | variable | UUID | required | UUID of the image to download | Response body is binary image. Empty bytes if image not found. + +### Recognize Faces from a Given Image + +To recognize faces from the uploaded image: + +```shell +curl -X POST "http://localhost:8000/api/v1/recognition/recognize?limit=&prediction_count=&det_prob_threshold=&face_plugins=&status=&detect_faces=" \ +-H "Content-Type: multipart/form-data" \ +-H "x-api-key: " \ +-F file= +``` + +| Element | Description | Type | Required | Notes | +|--------------------|-------------|---------|----------|------------------------------------------------------------------------------------------------------------------------------------------------| +| Content-Type | header | string | required | multipart/form-data | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| file | body | image | required | allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | +| limit | param | integer | optional | maximum number of faces on the image to be recognized. It recognizes the biggest faces first. Value of 0 represents no limit. Default value: 0 | +| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0. | +| prediction_count | param | integer | optional | maximum number of subject predictions per face. It returns the most similar subjects. Default value: 1 | +| face_plugins | param | string | optional | comma-separated slugs of face plugins. If empty, no additional information is returned. [Learn more](Face-services-and-plugins.md) | +| status | param | boolean | optional | if true includes system information like execution_time and plugin_version fields. Default value is false | +| detect_faces | param | boolean | optional | if false, CompreFace won't run a face detector. Instead, it will treat the image as a cropped face. Default value is true. Since 1.2 version | + +Response body on success: +```json +{ + "result" : [ { + "age" : { + "probability": 0.9308982491493225, + "high": 32, + "low": 25 + }, + "gender" : { + "probability": 0.9898611307144165, + "value": "female" + }, + "mask" : { + "probability": 0.9999470710754395, + "value": "without_mask" + }, + "embedding" : [ 9.424854069948196E-4, "...", -0.011415496468544006 ], + "box" : { + "probability" : 1.0, + "x_max" : 1420, + "y_max" : 1368, + "x_min" : 548, + "y_min" : 295 + }, + "landmarks" : [ [ 814, 713 ], [ 1104, 829 ], [ 832, 937 ], [ 704, 1030 ], [ 1017, 1133 ] ], + "subjects" : [ { + "similarity" : 0.97858, + "subject" : "subject1" + } ], + "execution_time" : { + "age" : 28.0, + "gender" : 26.0, + "detector" : 117.0, + "calculator" : 45.0, + "mask": 36.0 + } + } ], + "plugins_versions" : { + "age" : "agegender.AgeDetector", + "gender" : "agegender.GenderDetector", + "detector" : "facenet.FaceDetector", + "calculator" : "facenet.Calculator", + "mask": "facemask.MaskDetector" + } +} +``` + +| Element | Type | Description | +|----------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| age | object | detected age range. Return only if [age plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| gender | object | detected gender. Return only if [gender plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| pose | object | detected head pose. Return only if [pose plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| mask | object | detected mask. Return only if [face mask plugin](Face-services-and-plugins.md#face-plugins) is enabled. | +| embedding | array | face embeddings. Return only if [calculator plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| box | object | list of parameters of the bounding box for this face | +| probability | float | probability that a found face is actually a face | +| x_max, y_max, x_min, y_min | integer | coordinates of the frame containing the face | +| landmarks | array | list of the coordinates of the frame containing the face-landmarks. Return only if [landmarks plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| subjects | list | list of similar subjects with size of order by similarity | +| similarity | float | similarity that on that image predicted person | +| subject | string | name of the subject in Face Collection | +| execution_time | object | execution time of all plugins | +| plugins_versions | object | contains information about plugin versions | + + ### Verify Faces from a Given Image To compare faces from the uploaded images with the face in saved image ID: -```http request -curl -X POST "http://localhost:8000/api/v1/recognition/faces//verify? -limit=&det_prob_threshold=&face_plugins=&status=" \ +```shell +curl -X POST "http://localhost:8000/api/v1/recognition/faces//verify?limit=&det_prob_threshold=&face_plugins=&status=" \ -H "Content-Type: multipart/form-data" \ -H "x-api-key: " \ -F file= ``` -| Element | Description | Type | Required | Notes | -| ------------------ | ----------- | ------- | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | multipart/form-data | -| x-api-key | header | string | required | api key of the Face recognition service, created by the user | -| image_id | variable | UUID | required | UUID of the verifying face | -| file | body | image | required | allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | +| Element | Description | Type | Required | Notes | +|--------------------|-------------|---------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| Content-Type | header | string | required | multipart/form-data | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| image_id | variable | UUID | required | UUID of the verifying face | +| file | body | image | required | allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | | limit | param | integer | optional | maximum number of faces on the target image to be recognized. It recognizes the biggest faces first. Value of 0 represents no limit. Default value: 0 | -| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0. | -| face_plugins | param | string | optional | comma-separated slugs of face plugins. If empty, no additional information is returned. [Learn more](Face-services-and-plugins.md) | -| status | param | boolean | optional | if true includes system information like execution_time and plugin_version fields. Default value is false | +| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0. | +| face_plugins | param | string | optional | comma-separated slugs of face plugins. If empty, no additional information is returned. [Learn more](Face-services-and-plugins.md) | +| status | param | boolean | optional | if true includes system information like execution_time and plugin_version fields. Default value is false | Response body on success: ```json { "result": [ { - "age" : [ 25, 32 ], - "gender" : "female", + "age" : { + "probability": 0.9308982491493225, + "high": 32, + "low": 25 + }, + "gender" : { + "probability": 0.9898611307144165, + "value": "female" + }, + "mask" : { + "probability": 0.9999470710754395, + "value": "without_mask" + }, "embedding" : [ -0.049007344990968704, "...", -0.01753818802535534 ], "box" : { "probability" : 0.9997453093528748, @@ -445,7 +543,8 @@ Response body on success: "age" : 59.0, "gender" : 30.0, "detector" : 177.0, - "calculator" : 70.0 + "calculator" : 70.0, + "mask": 36.0 } } ], @@ -453,29 +552,32 @@ Response body on success: "age" : "agegender.AgeDetector", "gender" : "agegender.GenderDetector", "detector" : "facenet.FaceDetector", - "calculator" : "facenet.Calculator" + "calculator" : "facenet.Calculator", + "mask": "facemask.MaskDetector" } } ``` -| Element | Type | Description | -| ------------------------------ | ------- | ------------------------------------------------------------ | -| age | array | detected age range. Return only if [age plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| gender | string | detected gender. Return only if [gender plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| embedding | array | face embeddings. Return only if [calculator plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| box | object | list of parameters of the bounding box for this face | -| probability | float | probability that a found face is actually a face | -| x_max, y_max, x_min, y_min | integer | coordinates of the frame containing the face | -| landmarks | array | list of the coordinates of the frame containing the face-landmarks. Return only if [landmarks plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| similarity | float | similarity that on that image predicted person | -| execution_time | object | execution time of all plugins | -| plugins_versions | object | contains information about plugin versions | +| Element | Type | Description | +|----------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| age | object | detected age range. Return only if [age plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| gender | object | detected gender. Return only if [gender plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| pose | object | detected head pose. Return only if [pose plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| mask | object | detected mask. Return only if [face mask plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| embedding | array | face embeddings. Return only if [calculator plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| box | object | list of parameters of the bounding box for this face | +| probability | float | probability that a found face is actually a face | +| x_max, y_max, x_min, y_min | integer | coordinates of the frame containing the face | +| landmarks | array | list of the coordinates of the frame containing the face-landmarks. Return only if [landmarks plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| similarity | float | similarity that on that image predicted person | +| execution_time | object | execution time of all plugins | +| plugins_versions | object | contains information about plugin versions | ## Face Detection Service To detect faces from the uploaded image: -```http request +```shell curl -X POST "http://localhost:8000/api/v1/detection/detect?limit=&det_prob_threshold=&face_plugins=&status=" \ -H "Content-Type: multipart/form-data" \ -H "x-api-key: " \ @@ -483,22 +585,33 @@ curl -X POST "http://localhost:8000/api/v1/detection/detect?limit=&det_p ``` -| Element | Description | Type | Required | Notes | -| ------------------ | ----------- | ------- | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | multipart/form-data | -| x-api-key | header | string | required | api key of the Face Detection service, created by the user | -| file | body | image | required | image where to detect faces. Allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | +| Element | Description | Type | Required | Notes | +|--------------------|-------------|---------|----------|------------------------------------------------------------------------------------------------------------------------------------------------| +| Content-Type | header | string | required | multipart/form-data | +| x-api-key | header | string | required | api key of the Face Detection service, created by the user | +| file | body | image | required | image where to detect faces. Allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | | limit | param | integer | optional | maximum number of faces on the image to be recognized. It recognizes the biggest faces first. Value of 0 represents no limit. Default value: 0 | -| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0 | -| face_plugins | param | string | optional | comma-separated slugs of face plugins. If empty, no additional information is returned. [Learn more](Face-services-and-plugins.md) | -| status | param | boolean | optional | if true includes system information like execution_time and plugin_version fields. Default value is false | +| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0 | +| face_plugins | param | string | optional | comma-separated slugs of face plugins. If empty, no additional information is returned. [Learn more](Face-services-and-plugins.md) | +| status | param | boolean | optional | if true includes system information like execution_time and plugin_version fields. Default value is false | Response body on success: ```json { "result" : [ { - "age" : [ 25, 32 ], - "gender" : "female", + "age" : { + "probability": 0.9308982491493225, + "high": 32, + "low": 25 + }, + "gender" : { + "probability": 0.9898611307144165, + "value": "female" + }, + "mask" : { + "probability": 0.9999470710754395, + "value": "without_mask" + }, "embedding" : [ -0.03027934394776821, "...", -0.05117142200469971 ], "box" : { "probability" : 0.9987509250640869, @@ -512,35 +625,39 @@ Response body on success: "age" : 30.0, "gender" : 26.0, "detector" : 130.0, - "calculator" : 49.0 + "calculator" : 49.0, + "mask": 36.0 } } ], "plugins_versions" : { "age" : "agegender.AgeDetector", "gender" : "agegender.GenderDetector", "detector" : "facenet.FaceDetector", - "calculator" : "facenet.Calculator" + "calculator" : "facenet.Calculator", + "mask": "facemask.MaskDetector" } } ``` -| Element | Type | Description | -| ------------------------------ | ------- | ------------------------------------------------------------ | -| age | array | detected age range. Return only if [age plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| gender | string | detected gender. Return only if [gender plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| embedding | array | face embeddings. Return only if [calculator plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| box | object | list of parameters of the bounding box for this face (on processedImage) | -| probability | float | probability that a found face is actually a face (on processedImage) | -| x_max, y_max, x_min, y_min | integer | coordinates of the frame containing the face (on processedImage) | -| landmarks | array | list of the coordinates of the frame containing the face-landmarks. Return only if [landmarks plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| execution_time | object | execution time of all plugins | -| plugins_versions | object | contains information about plugin versions | +| Element | Type | Description | +|----------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| age | object | detected age range. Return only if [age plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| gender | object | detected gender. Return only if [gender plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| pose | object | detected head pose. Return only if [pose plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| mask | object | detected mask. Return only if [face mask plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| embedding | array | face embeddings. Return only if [calculator plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| box | object | list of parameters of the bounding box for this face (on processedImage) | +| probability | float | probability that a found face is actually a face (on processedImage) | +| x_max, y_max, x_min, y_min | integer | coordinates of the frame containing the face (on processedImage) | +| landmarks | array | list of the coordinates of the frame containing the face-landmarks. Return only if [landmarks plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| execution_time | object | execution time of all plugins | +| plugins_versions | object | contains information about plugin versions | ## Face Verification Service To compare faces from given two images: -```http request +```shell curl -X POST "http://localhost:8000/api/v1/verification/verify?limit=&prediction_count=&det_prob_threshold=&face_plugins=&status=" \ -H "Content-Type: multipart/form-data" \ -H "x-api-key: " \ @@ -549,24 +666,35 @@ curl -X POST "http://localhost:8000/api/v1/verification/verify?limit=&pr ``` -| Element | Description | Type | Required | Notes | -| ------------------ | ----------- | ------- | -------- | ------------------------------------------------------------ | -| Content-Type | header | string | required | multipart/form-data | -| x-api-key | header | string | required | api key of the Face verification service, created by the user | -| source_image | body | image | required | file to be verified. Allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | -| target_image | body | image | required | reference file to check the source file. Allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | +| Element | Description | Type | Required | Notes | +|--------------------|-------------|---------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| Content-Type | header | string | required | multipart/form-data | +| x-api-key | header | string | required | api key of the Face verification service, created by the user | +| source_image | body | image | required | file to be verified. Allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | +| target_image | body | image | required | reference file to check the source file. Allowed image formats: jpeg, jpg, ico, png, bmp, gif, tif, tiff, webp. Max size is 5Mb | | limit | param | integer | optional | maximum number of faces on the target image to be recognized. It recognizes the biggest faces first. Value of 0 represents no limit. Default value: 0 | -| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0. | -| face_plugins | param | string | optional | comma-separated slugs of face plugins. If empty, no additional information is returned. [Learn more](Face-services-and-plugins.md) | -| status | param | boolean | optional | if true includes system information like execution_time and plugin_version fields. Default value is false | +| det_prob_threshold | param | string | optional | minimum required confidence that a recognized face is actually a face. Value is between 0.0 and 1.0. | +| face_plugins | param | string | optional | comma-separated slugs of face plugins. If empty, no additional information is returned. [Learn more](Face-services-and-plugins.md) | +| status | param | boolean | optional | if true includes system information like execution_time and plugin_version fields. Default value is false | Response body on success: ```json { "result" : [{ "source_image_face" : { - "age" : [ 25, 32 ], - "gender" : "female", + "age" : { + "probability": 0.9308982491493225, + "high": 32, + "low": 25 + }, + "gender" : { + "probability": 0.9898611307144165, + "value": "female" + }, + "mask" : { + "probability": 0.9999470710754395, + "value": "without_mask" + }, "embedding" : [ -0.0010271212086081505, "...", -0.008746841922402382 ], "box" : { "probability" : 0.9997453093528748, @@ -580,13 +708,25 @@ Response body on success: "age" : 85.0, "gender" : 51.0, "detector" : 67.0, - "calculator" : 116.0 + "calculator" : 116.0, + "mask": 36.0 } }, "face_matches": [ { - "age" : [ 25, 32 ], - "gender" : "female", + "age" : { + "probability": 0.9308982491493225, + "high": 32, + "low": 25 + }, + "gender" : { + "probability": 0.9898611307144165, + "value": "female" + }, + "mask" : { + "probability": 0.9999470710754395, + "value": "without_mask" + }, "embedding" : [ -0.049007344990968704, "...", -0.01753818802535534 ], "box" : { "probability" : 0.99975, @@ -601,33 +741,37 @@ Response body on success: "age" : 59.0, "gender" : 30.0, "detector" : 177.0, - "calculator" : 70.0 + "calculator" : 70.0, + "mask": 36.0 } }], "plugins_versions" : { "age" : "agegender.AgeDetector", "gender" : "agegender.GenderDetector", "detector" : "facenet.FaceDetector", - "calculator" : "facenet.Calculator" + "calculator" : "facenet.Calculator", + "mask": "facemask.MaskDetector" } }] } ``` -| Element | Type | Description | -| ------------------------------ | ------- | ------------------------------------------------------------ | -| source_image_face | object | additional info about source image face | -| face_matches | array | result of face verification | -| age | array | detected age range. Return only if [age plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| gender | string | detected gender. Return only if [gender plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| embedding | array | face embeddings. Return only if [calculator plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| box | object | list of parameters of the bounding box for this face | -| probability | float | probability that a found face is actually a face | -| x_max, y_max, x_min, y_min | integer | coordinates of the frame containing the face | -| landmarks | array | list of the coordinates of the frame containing the face-landmarks. Return only if [landmarks plugin](Face-services-and-plugins.md#face-plugins) is enabled | -| similarity | float | similarity between this face and the face on the source image | -| execution_time | object | execution time of all plugins | -| plugins_versions | object | contains information about plugin versions | +| Element | Type | Description | +|----------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| source_image_face | object | additional info about source image face | +| face_matches | array | result of face verification | +| age | object | detected age range. Return only if [age plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| gender | object | detected gender. Return only if [gender plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| pose | object | detected head pose. Return only if [pose plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| mask | object | detected mask. Return only if [face mask plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| embedding | array | face embeddings. Return only if [calculator plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| box | object | list of parameters of the bounding box for this face | +| probability | float | probability that a found face is actually a face | +| x_max, y_max, x_min, y_min | integer | coordinates of the frame containing the face | +| landmarks | array | list of the coordinates of the frame containing the face-landmarks. Return only if [landmarks plugin](Face-services-and-plugins.md#face-plugins) is enabled | +| similarity | float | similarity between this face and the face on the source image | +| execution_time | object | execution time of all plugins | +| plugins_versions | object | contains information about plugin versions | @@ -641,7 +785,7 @@ The name of the JSON parameter coincides with the name of the `multipart/form-da ### Add an Example of a Subject, Base64 Full description [here](#add-an-example-of-a-subject). -```http request +```shell curl -X POST "http://localhost:8000/api/v1/recognition/faces?subject=&det_prob_threshold=" \ -H "Content-Type: application/json" \ -H "x-api-key: " \ @@ -651,7 +795,7 @@ curl -X POST "http://localhost:8000/api/v1/recognition/faces?subject=&d ### Recognize Faces from a Given Image, Base64 Full description [here](#recognize-faces-from-a-given-image). -```http request +```shell curl -X POST "http://localhost:8000/api/v1/recognition/recognize?limit=&prediction_count=&det_prob_threshold=&face_plugins=&status=" \ -H "Content-Type: application/json" \ -H "x-api-key: " \ @@ -661,7 +805,7 @@ curl -X POST "http://localhost:8000/api/v1/recognition/recognize?limit=& ### Verify Faces from a Given Image, Base64 Full description [here](#verify-faces-from-a-given-image). -```http request +```shell curl -X POST "http://localhost:8000/api/v1/recognition/faces//verify? limit=&det_prob_threshold=&face_plugins=&status=" \ -H "Content-Type: application/json" \ @@ -672,7 +816,7 @@ limit=&det_prob_threshold=&face_plugins=&det_prob_threshold=&face_plugins=&status=" \ -H "Content-Type: application/json" \ -H "x-api-key: " \ @@ -682,10 +826,135 @@ curl -X POST "http://localhost:8000/api/v1/detection/detect?limit=&det_p ### Face Verification Service, Base64 Full description [here](#face-verification-service). -```http request +```shell curl -X POST "http://localhost:8000/api/v1/verification/verify?limit=&prediction_count=&det_prob_threshold=&face_plugins=&status=" \ -H "Content-Type: application/json" \ -H "x-api-key: " \ -d {"source_image": "", "target_image": ""} ``` + + +## Recognition and verification using embedding +`since 1.2.0 version` + +You can use computed embedding to perform recognition and verification. To obtain embedding, you can +use [calculator plugin](https://github.com/exadel-inc/CompreFace/blob/EFRS-1333_ability_to_send_embeddings_instead_of_the_image_for_recognition/docs/Face-services-and-plugins.md#face-plugins) +in each Face service. +The base rule is to use `Content-Type: application/json` header and send JSON in the body. + +### Recognize Faces from a Given Image, Embedding +The service is used to determine similarities between input embeddings and embeddings within the Face Collection. An example: + +```shell +curl -X POST "http://localhost:8000/api/v1/recognition/embeddings/recognize?prediction_count=" \ +-H "Content-Type: application/json" \ +-H "x-api-key: " \ +-d {"embeddings": [[], ...]} +``` + +| Element | Description | Type | Required | Notes | +|------------------|-------------|---------|----------|-----------------------------------------------------------------------------------------------------------------| +| Content-Type | header | string | required | application/json | +| x-api-key | header | string | required | an api key of the Face recognition service, created by the user | +| embeddings | body | array | required | an input embeddings. The length depends on the model (e.g. 512 or 128) | +| prediction_count | param | integer | optional | the maximum number of subject predictions per embedding. It returns the most similar subjects. Default value: 1 | + +Response body on success: +```json +{ + "result": [ + { + "embedding": [0.0627421774604647, "...", -0.0236684433507126], + "similarities": [ + { + "subject": "John", + "similarity": 0.55988 + }, + "..." + ] + }, + "..." + ] +} +``` + +| Element | Type | Description | +|--------------|--------|--------------------------------------------------------------------------------------------| +| result | array | an array that contains all the results | +| embedding | array | an input embedding | +| similarities | array | an array that contains results of similarity between the embedding and the input embedding | +| subject | string | a subject in which the similar embedding was found | +| similarity | float | a similarity between the embedding and the input embedding | + +### Verify Faces from a Given Image, Embedding +The endpoint is used to compare input embeddings to the embedding stored in Face Collection. An example: + +```shell +curl -X POST "http://localhost:8000/api/v1/recognition/embeddings/faces/{image_id}/verify" \ +-H "Content-Type: application/json" \ +-H "x-api-key: " \ +-d {"embeddings": [[], ...]} +``` + +| Element | Description | Type | Required | Notes | +|--------------|-------------|--------|----------|------------------------------------------------------------------------| +| Content-Type | header | string | required | application/json | +| x-api-key | header | string | required | api key of the Face recognition service, created by the user | +| embeddings | body | array | required | an input embeddings. The length depends on the model (e.g. 512 or 128) | +| image_id | variable | UUID | required | an id of the source embedding within the Face Collection | + +Response body on success: +```json +{ + "result": [ + { + "embedding": [0.0627421774604647, "...", -0.0236684433507126], + "similarity": 0.55988 + }, + "..." + ] +} +``` + +| Element | Type | Description | +|-------------|--------|------------------------------------------------------------------------------| +| result | array | an array that contains all the results | +| embedding | array | a source embedding which we are comparing to embedding from Face Collection | +| similarity | float | a similarity between the source embedding and embedding from Face Collection | + +### Face Verification Service, Embedding +The service is used to determine similarities between an input source embedding and input target embeddings. An example: + +```shell +curl -X POST "http://localhost:8000/api/v1/verification/embeddings/verify" \ +-H "Content-Type: application/json" \ +-H "x-api-key: " \ +-d {"source": [], "targets": [[], ...]} +``` + +| Element | Description | Type | Required | Notes | +|------------------|-------------|---------|----------|--------------------------------------------------------------------------------------| +| Content-Type | header | string | required | application/json | +| x-api-key | header | string | required | api key of the Face verification service, created by the user | +| source | body | array | required | an input embeddings. The length depends on the model (e.g. 512 or 128) | +| targets | body | array | required | an array of the target embeddings. The length depends on the model (e.g. 512 or 128) | + +Response body on success: +```json +{ + "result": [ + { + "embedding": [0.0627421774604647, "...", -0.0236684433507126], + "similarity": 0.55988 + }, + "..." + ] +} +``` + +| Element | Type | Description | +|-------------|--------|--------------------------------------------------------------------| +| result | array | an array that contains all the results | +| embedding | array | a target embedding which we are comparing to source embedding | +| similarity | float | a similarity between the source embedding and the target embedding | diff --git a/docs/User-Roles-System.md b/docs/User-Roles-System.md index cfa8bec7a5..c21fe50732 100644 --- a/docs/User-Roles-System.md +++ b/docs/User-Roles-System.md @@ -1,47 +1,67 @@ # User Roles System -CompreFace roles system consists of two types of roles - global roles and application roles. -The users with these roles have different responsibilities, -so we recommend that you delimit such users and follow our recommendations to avoid giving too much access to sensitive information. -Of course in small teams and at your own risk you can ignore these recommendations. +CompreFace roles system consists of two types of roles - global roles +and application roles. The users with these roles have different +responsibilities, so we recommend that you delimit such users and follow +our recommendations to avoid giving too much access to sensitive +information. Of course, in small teams and at your own risk, you can +ignore these recommendations. ## Global Roles -Global roles define what permissions you have in the system itself, and the main responsibility of such users is to maintain the system itself. -We recommend that the most permissive (owner and administrator) roles be added to technical support employees. -Then there is no reason to add such users to applications as they still will have all permissions within the application. +Global roles define what permissions you have in the system itself, and +the primary responsibility of such users is to maintain the system +itself. Therefore, we recommend adding the most permissive (owner and +administrator) roles to technical support employees. Then there is no +reason to add such users to applications as they still have all +permissions within the application. -In CompreFace, the first user automatically receives the global owner role and has rights for any operation within CompreFace - -managing users, and creating and managing applications. -The only restriction for the global owner is that such a user can’t delete themselves from the system, -so the user will have to assign the global owner role to somebody else and then remove themselves from the system. +In CompreFace, the first user automatically receives the global owner +role and has rights for any operation within CompreFace - managing users +and creating and managing applications. The only restriction for the +global owner is that such a user can't delete themselves from the +system, so the user has to assign the global owner role to somebody else +and then remove themselves from the system. -Users with the global administrator role have the same permissions as users with the global owner role. -The only difference is that such users can’t manage the user with the global owner role. -We recommend reducing users with such a role to the minimum number required to maintain the system. +Users with the global administrator role have the same permissions as +users with the global owner role. The only difference is that such users +can\'t manage the user with the global owner role. We recommend reducing +users with such a role to the minimum number required to maintain the +system. -All new users are automatically assigned the global user role. These users can’t create applications, -can access only the applications to which they were added, and can’t manage other users. -These users use CompreFace for face recognition and are part of the development team; -they are not responsible for managing other users and their permissions. +All new users are automatically assigned the global user role. These +users can't create applications, can access only the applications they +were added to, and can't manage other users. These users use CompreFace +for face recognition and are part of the development team; they are not +responsible for managing other users and their permissions. ## Application Roles -Application roles define the role of the user within an application, -and the main responsibility of such users is to develop applications into which they are going to integrate CompreFace. -We recommend that the most permissive roles (owner and administrator) be added as project managers and team leads, -as they are responsible for the application. We also recommend that all application users have the global user role. -To become a member of an application team, users with a global user role need to be added to the application directly by the global owner, -global administrator, or application owner. +Application roles define the user's role within an application, and the +primary responsibility of such users is to develop applications into +which they are going to integrate CompreFace. We recommend that the most +permissive roles (owner and administrator) were added as project +managers and team leads, as they are responsible for the application. We +also recommend that all application users have the global user role. To +become a member of an application team, users with a global user role +need to be added to the application directly by the global owner, global +administrator, or application owner. -The user that creates an application automatically receives the application owner role and has rights for any operation within the application - -managing the application and its users, and creating and managing [Face Services](Face-services-and-plugins.md). -The only restriction for the application owner is that they can’t delete themselves from the application, -so they have to assign the application owner role to somebody else before deleting themselves. +The user that creates an application automatically receives the +application owner role and has rights for any operation within the +application - managing the application and its users and creating and +managing [Face Services](Face-services-and-plugins.md). The only +restriction for the application owner is that they can't delete +themselves from the application, so they have to assign the application +owner role to somebody else before deleting themselves. -Users with the application administrator role (global user role + application admin role) -can create and manage [Face Services](Face-services-and-plugins.md) but can’t manage an application and its users. +Users with the application administrator role (global user role + +application administrator role) can create and manage [Face +Services](Face-services-and-plugins.md) but can't manage an application +and its users. -Users with the application user role can’t manage anything in the application. This is the least permissive role -(global user role + application user role), but this provides enough information to integrate CompreFace with any other application, -so we recommend that most CompreFace users have this role. \ No newline at end of file +Users with the application user role can't manage anything in the +application. This is the least permissive role (global user role + +application user role), but this provides enough information to +integrate CompreFace with any other application, so we recommend that +most CompreFace users have this role. diff --git a/embedding-calculator/Dockerfile b/embedding-calculator/Dockerfile index 2fb58e9bd3..ff57eae520 100644 --- a/embedding-calculator/Dockerfile +++ b/embedding-calculator/Dockerfile @@ -1,15 +1,48 @@ ARG BASE_IMAGE -FROM ${BASE_IMAGE:-python:3.7-slim} +FROM ${BASE_IMAGE:-python:3.8-slim-bullseye} RUN apt-get update && apt-get install -y build-essential cmake git wget unzip \ curl yasm pkg-config libswscale-dev libtbb2 libtbb-dev libjpeg-dev \ - libpng-dev libtiff-dev libavformat-dev libpq-dev libfreeimage3 \ + libpng-dev libtiff-dev libavformat-dev libpq-dev libfreeimage3 python3-opencv \ + libaec-dev libblosc-dev libbrotli-dev libbz2-dev libgif-dev libopenjp2-7-dev \ + liblcms2-dev libcharls-dev libjxr-dev liblz4-dev libcfitsio-dev libpcre3 libpcre3-dev \ + libsnappy-dev libwebp-dev libzopfli-dev libzstd-dev \ && rm -rf /var/lib/apt/lists/* +# Dependencies for imagecodecs +WORKDIR /tmp + +# brunsli +RUN git clone --depth=1 --shallow-submodules --recursive -b v0.1 https://github.com/google/brunsli && \ + cd brunsli && \ + cmake -DCMAKE_BUILD_TYPE=Release . && \ + make -j$(nproc) install && \ + rm -rf /tmp/brunsli + +# libjxl +RUN git clone --depth=1 --shallow-submodules --recursive -b v0.7.0 https://github.com/libjxl/libjxl && \ + cd libjxl && \ + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF . && \ + make -j$(nproc) install && \ + rm -rf /tmp/libjxl + +# zfp +RUN git clone --depth=1 -b 0.5.5 https://github.com/LLNL/zfp && \ + cd zfp && \ + mkdir build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ + make -j$(nproc) install && \ + rm -rf /tmp/zfp +# End imagecodecs dependencies + # install common python packages SHELL ["/bin/bash", "-c"] WORKDIR /app/ml COPY requirements.txt . +# Ensure numpy is installed first as imagecodecs doesn't declare dependencies correctly until 2022.9.26, +# which is not compatible with Python 3.7 +RUN pip --no-cache-dir install $(grep ^numpy requirements.txt) RUN pip --no-cache-dir install -r requirements.txt ARG BE_VERSION @@ -28,7 +61,7 @@ ARG GPU_IDX=-1 ENV GPU_IDX=$GPU_IDX INTEL_OPTIMIZATION=$INTEL_OPTIMIZATION ARG FACE_DETECTION_PLUGIN="facenet.FaceDetector" ARG CALCULATION_PLUGIN="facenet.Calculator" -ARG EXTRA_PLUGINS="facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector" +ARG EXTRA_PLUGINS="facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector,facenet.PoseEstimator" ENV FACE_DETECTION_PLUGIN=$FACE_DETECTION_PLUGIN CALCULATION_PLUGIN=$CALCULATION_PLUGIN \ EXTRA_PLUGINS=$EXTRA_PLUGINS COPY src src @@ -47,6 +80,4 @@ RUN if [ -z $SKIP_TESTS ]; then pytest -m "not performance" /app/ml/src; fi EXPOSE 3000 COPY uwsgi.ini . -ENV UWSGI_PROCESSES=${UWSGI_PROCESSES:-2} -ENV UWSGI_THREADS=1 CMD ["uwsgi", "--ini", "uwsgi.ini"] diff --git a/embedding-calculator/Makefile b/embedding-calculator/Makefile index 8f9b540bf0..c31fcb053c 100755 --- a/embedding-calculator/Makefile +++ b/embedding-calculator/Makefile @@ -3,19 +3,30 @@ SHELL := /bin/bash .PHONY := default up build-images build-cuda IMAGE := ${DOCKER_REGISTRY}compreface-core -CUDA_IMAGE = $(IMAGE)-base:base-cuda100-py37 +CUDA_IMAGE := $(IMAGE)-base:base-cuda118-py38 +APPERY_ARG := --build-arg APPERY_API_KEY=${APPERY_API_KEY} MOBILENET_BUILD_ARGS := --build-arg FACE_DETECTION_PLUGIN=insightface.FaceDetector@retinaface_mnet025_v1 \ --build-arg CALCULATION_PLUGIN=insightface.Calculator@arcface_mobilefacenet \ - --build-arg EXTRA_PLUGINS=insightface.LandmarksDetector,insightface.GenderDetector,insightface.AgeDetector + --build-arg EXTRA_PLUGINS=insightface.LandmarksDetector,insightface.GenderDetector,insightface.AgeDetector,insightface.facemask.MaskDetector,insightface.PoseEstimator \ + $(APPERY_ARG) ARCFACE_r100_BUILD_ARGS := --build-arg FACE_DETECTION_PLUGIN=insightface.FaceDetector@retinaface_r50_v1 \ --build-arg CALCULATION_PLUGIN=insightface.Calculator@arcface-r100-msfdrop75 \ - --build-arg EXTRA_PLUGINS=insightface.LandmarksDetector,insightface.GenderDetector,insightface.AgeDetector + --build-arg EXTRA_PLUGINS=insightface.LandmarksDetector,insightface.GenderDetector,insightface.AgeDetector,insightface.facemask.MaskDetector,insightface.PoseEstimator \ + $(APPERY_ARG) FACENET_BUILD_ARGS := --build-arg FACE_DETECTION_PLUGIN=facenet.FaceDetector \ --build-arg CALCULATION_PLUGIN=facenet.Calculator \ - --build-arg EXTRA_PLUGINS=facenet.LandmarksDetector,agegender.GenderDetector,agegender.AgeDetector + --build-arg EXTRA_PLUGINS=facenet.LandmarksDetector,agegender.GenderDetector,agegender.AgeDetector,facenet.facemask.MaskDetector,facenet.PoseEstimator \ + $(APPERY_ARG) + +CORAL_BUILD_ARGS := --build-arg FACE_DETECTION_PLUGIN=facenet.coralmtcnn.FaceDetector \ + --build-arg CALCULATION_PLUGIN=facenet.coralmtcnn.Calculator \ + --build-arg EXTRA_PLUGINS=facenet.LandmarksDetector,agegender.GenderDetector,agegender.AgeDetector,facenet.facemask.MaskDetector,facenet.PoseEstimator \ + $(APPERY_ARG) + +GPU_ARGS := --build-arg GPU_IDX=0 --build-arg BASE_IMAGE=$(CUDA_IMAGE) define get_from_remote_tgz mkdir -p $(2) @@ -40,20 +51,24 @@ build-cuda: docker build . --file gpu.Dockerfile --tag $(CUDA_IMAGE) build-images: build-cuda - docker build . -t $(IMAGE):$(VERSION)-facenet + docker build . -t $(IMAGE):$(VERSION)-facenet $(FACENET_BUILD_ARGS) docker build . -t $(IMAGE):$(VERSION)-mobilenet $(MOBILENET_BUILD_ARGS) - docker build . -t $(IMAGE):$(VERSION)-mobilenet-gpu $(MOBILENET_BUILD_ARGS) --build-arg GPU_IDX=0 --build-arg BASE_IMAGE=$(CUDA_IMAGE) + docker build . -t $(IMAGE):$(VERSION)-mobilenet-gpu $(MOBILENET_BUILD_ARGS) $(GPU_ARGS) docker build . -t $(IMAGE):$(VERSION)-arcface-r100 $(ARCFACE_r100_BUILD_ARGS) - docker build . -t $(IMAGE):$(VERSION)-arcface-r100-gpu $(ARCFACE_r100_BUILD_ARGS) --build-arg GPU_IDX=0 --build-arg BASE_IMAGE=$(CUDA_IMAGE) + docker build . -t $(IMAGE):$(VERSION)-arcface-r100-gpu $(ARCFACE_r100_BUILD_ARGS) $(GPU_ARGS) + docker build . -t $(IMAGE):$(VERSION)-facenet-tpu --file tpu.Dockerfile $(CORAL_BUILD_ARGS) build-images-cpu: - docker build . -t $(IMAGE):$(VERSION)-facenet + docker build . -t $(IMAGE):$(VERSION)-facenet $(FACENET_BUILD_ARGS) docker build . -t $(IMAGE):$(VERSION)-mobilenet $(MOBILENET_BUILD_ARGS) docker build . -t $(IMAGE):$(VERSION)-arcface-r100 $(ARCFACE_r100_BUILD_ARGS) build-images-gpu: build-cuda - docker build . -t $(IMAGE):$(VERSION)-mobilenet-gpu $(MOBILENET_BUILD_ARGS) --build-arg GPU_IDX=0 --build-arg BASE_IMAGE=$(CUDA_IMAGE) - docker build . -t $(IMAGE):$(VERSION)-arcface-r100-gpu $(ARCFACE_r100_BUILD_ARGS) --build-arg GPU_IDX=0 --build-arg BASE_IMAGE=$(CUDA_IMAGE) + docker build . -t $(IMAGE):$(VERSION)mobilenet-gpu $(MOBILENET_BUILD_ARGS) $(GPU_ARGS) + docker build . -t $(IMAGE):$(VERSION)arcface-r100-gpu $(ARCFACE_r100_BUILD_ARGS) $(GPU_ARGS) + +build-images-tpu: + docker build . -t $(IMAGE):$(VERSION)-facenet-tpu --file tpu.Dockerfile $(CORAL_BUILD_ARGS) up: docker run -p3000:3000 embedding-calculator @@ -64,3 +79,4 @@ tools/tmp: tools/benchmark_detection/tmp: $(call get_from_remote_tgz,http://tamaraberg.com/faceDataset/originalPics.tar.gz,tools/benchmark_detection/tmp/originalPics) $(call get_from_remote_tgz,http://vis-www.cs.umass.edu/fddb/FDDB-folds.tgz,tools/benchmark_detection/tmp) + diff --git a/embedding-calculator/README.md b/embedding-calculator/README.md index fc066349b6..3905a4fda0 100644 --- a/embedding-calculator/README.md +++ b/embedding-calculator/README.md @@ -43,7 +43,7 @@ $ docker run -p 3000:3000 exadel/compreface-core:latest ##### Build Builds container (also runs main tests during the build): ``` -$ docker build -t embedding-calculator +$ docker build -t embedding-calculator . ``` To skip tests during build, use: ``` @@ -97,13 +97,15 @@ Pass to `EXTRA_PLUGINS` comma-separated names of plugins. |------------------------------------|----------------|-------------|------------|-------------| | agegender.AgeDetector | age | agegender | Tensorflow | | | agegender.GenderDetector | gender | agegender | Tensorflow | | -| insightface.AgeDetector | age | insightface | MXNet | + | -| insightface.GenderDetector | gender | insightface | MXNet | + | -| facenet.LandmarksDetector | landmarks | Facenet | Tensorflow | + | -| insightface.LandmarksDetector | landmarks | insightface | MXNet | + | -| insightface.Landmarks2d106Detector | landmarks2d106 | insightface | MXNet | + | -| facenet.facemask.MaskDetector | mask | facemask | Tensorflow | + | -| insightface.facemask.MaskDetector | mask | facemask | MXNet | + | +| insightface.AgeDetector | age | insightface | MXNet | + | +| insightface.GenderDetector | gender | insightface | MXNet | + | +| facenet.LandmarksDetector | landmarks | Facenet | Tensorflow | + | +| insightface.LandmarksDetector | landmarks | insightface | MXNet | + | +| insightface.Landmarks2d106Detector | landmarks2d106 | insightface | MXNet | + | +| facenet.facemask.MaskDetector | mask | facemask | Tensorflow | + | +| insightface.facemask.MaskDetector | mask | facemask | MXNet | + | +| facenet.PoseEstimator | pose | Facenet | Tensorflow | + | +| insightface.PoseEstimator | pose | insightface | MXNet | + | Notes: * `facenet.LandmarksDetector` and `insightface.LandmarksDetector` extract landmarks @@ -116,7 +118,7 @@ Notes: ``` FACE_DETECTION_PLUGIN=facenet.FaceDetector CALCULATION_PLUGIN=facenet.Calculator -EXTRA_PLUGINS=agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector +EXTRA_PLUGINS=facenet.LandmarksDetector,agegender.GenderDetector,agegender.AgeDetector,facenet.facemask.MaskDetector,facenet.PoseEstimator ``` #### Pre-trained models @@ -162,7 +164,14 @@ There are two build arguments for optimization: * `INTEL_OPTIMIZATION` - enable Intel MKL optimization (true/false) -##### NVIDIA Runtime +##### GPU Setup (Windows): +1. Install or update Docker Desktop. +2. Make sure that you have Windows version 21H2 or higher. +3. Update your NVIDIA drivers. +4. Install or update WSL2 Linux kernel. +5. Make sure the WSL2 backend is enabled in Docker Desktop. + +##### GPU Setup (Linux): Install the nvidia-docker2 package and dependencies on the host machine: ``` @@ -171,7 +180,7 @@ sudo apt-get install -y nvidia-docker2 sudo systemctl restart docker ``` -Build and run with enabled gpu +##### Build and run with enabled gpu ``` docker build . -t embedding-calculator-cuda -f gpu.Dockerfile docker build . -t embedding-calculator-gpu --build-arg GPU_IDX=0 --build-arg BASE_IMAGE=embedding-calculator-cuda diff --git a/embedding-calculator/gpu.Dockerfile b/embedding-calculator/gpu.Dockerfile index b43511f5a4..93de659edd 100644 --- a/embedding-calculator/gpu.Dockerfile +++ b/embedding-calculator/gpu.Dockerfile @@ -1,70 +1,27 @@ -ARG UBUNTU_VERSION=18.04 +ARG BASE_IMAGE +FROM ${BASE_IMAGE:-nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu20.04} -ARG ARCH= -ARG CUDA=10.0 -FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base -# ARCH and CUDA are specified again because the FROM directive resets ARGs -# (but their default value is retained if set previously) -ARG ARCH -ARG CUDA -ARG CUDNN=7.6.4.38-1 -ARG CUDNN_MAJOR_VERSION=7 -ARG LIB_DIR_PREFIX=x86_64 -ARG LIBNVINFER=6.0.1-1 -ARG LIBNVINFER_MAJOR_VERSION=6 -ENV CUDA=$CUDA +ENV DEBIAN_FRONTEND=noninteractive +ENV CUDA=11.8 -# Needed for string substitution -SHELL ["/bin/bash", "-c"] -# Pick up some TF dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ - cuda-command-line-tools-${CUDA/./-} \ - # There appears to be a regression in libcublas10=10.2.2.89-1 which - # prevents cublas from initializing in TF. See - # https://github.com/tensorflow/tensorflow/issues/9489#issuecomment-562394257 - libcublas10=10.2.1.243-1 \ - cuda-nvrtc-${CUDA/./-} \ - cuda-cufft-${CUDA/./-} \ - cuda-curand-${CUDA/./-} \ - cuda-cusolver-${CUDA/./-} \ - cuda-cusparse-${CUDA/./-} \ - curl \ - libcudnn7=${CUDNN}+cuda${CUDA} \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - pkg-config \ software-properties-common \ - unzip - -# Install TensorRT if not building for PowerPC -RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \ - libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/*; } - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH - -# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure -# dynamic linker run-time bindings -RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ - && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ - && ldconfig + curl \ + pkg-config \ + unzip \ + python3-dev \ + python3-distutils \ + && rm -rf /var/lib/apt/lists/* # See http://bugs.python.org/issue19846 ENV LANG C.UTF-8 -ARG PYTHON=python3.7 -RUN add-apt-repository ppa:deadsnakes/ppa && apt-get update && apt-get install -y ${PYTHON} libpython3.7-dev libgl1-mesa-glx -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && ${PYTHON} get-pip.py -RUN ${PYTHON} -m pip --no-cache-dir install --upgrade pip setuptools +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py +RUN python3 -m pip --no-cache-dir install --upgrade pip setuptools # Some TF tools expect a "python" binary -RUN ln -s $(which $PYTHON) /usr/local/bin/python - +RUN ln -s $(which python3) /usr/local/bin/python # Variables for Tensorflow ENV TF_FORCE_GPU_ALLOW_GROWTH=true @@ -75,6 +32,6 @@ ENV MXNET_ENGINE_TYPE=ThreadedEnginePerDevice MXNET_CUDNN_AUTOTUNE_DEFAULT=0 # No access to GPU devices in the build stage, so skip tests ENV SKIP_TESTS=1 + # The number of processes depends on GPU memory. # Keep in mind that one uwsgi process with InsightFace consumes about 2.5GB memory -ENV UWSGI_PROCESSES=1 \ No newline at end of file diff --git a/embedding-calculator/requirements.txt b/embedding-calculator/requirements.txt index c3f6fb4c9c..da86a5ab4a 100644 --- a/embedding-calculator/requirements.txt +++ b/embedding-calculator/requirements.txt @@ -3,21 +3,24 @@ cached-property==1.5.2 colour==0.1.5 flasgger==0.9.5 Flask==1.1.2 -gdown~=3.12 +itsdangerous==2.0.1 # itsdangerous is a Flask dependency, update it with Flask +jinja2<3.1.0 # jinja2 is a Flask dependency, update it with Flask +gdown~=4.5.4 Werkzeug==1.0.1 +PyYAML==5.4.1 # tests -mock~=4.0.2 -pytest~=6.1.2 -pytest-mock~=3.3.1 -requests~=2.24.0 -pylama~=7.7.1 +mock +pytest +pytest-mock +requests +pylama # dependencies for both scanner backends -Pillow~=8.0.1 +Pillow~=8.3.2 imagecodecs~=2020.5.30 numpy~=1.19.5 -scipy~=1.5.4 +scipy~=1.4.1 opencv-python~=4.4.0 scikit-learn~=0.23.2 scikit-image~=0.17.2 @@ -27,4 +30,4 @@ joblib~=0.17.0 uWSGI==2.0.19 # for successful import with tensorflow -mtcnn~=0.1.0 \ No newline at end of file +# mtcnn~=0.1.0 diff --git a/embedding-calculator/sample_images/annotations.py b/embedding-calculator/sample_images/annotations.py index 9ed428d60c..c82653d9c1 100644 --- a/embedding-calculator/sample_images/annotations.py +++ b/embedding-calculator/sample_images/annotations.py @@ -40,7 +40,7 @@ def __iter__(self): Row('013_4.jpg', [(275, 195), (335, 192), (423, 200), (492, 195)]), Row('014_5.jpg', [(98, 283), (207, 265), (405, 175), (602, 270), (687, 305)]), Row('015_6.jpg', [(161, 229), (267, 266), (350, 281), (450, 271), (555, 264), (635, 250)]), - Row('016_8.jpg', [(197, 277), (261, 171), (261, 292), (355, 282), (437, 212), (452, 288), (517, 163), + Row('016_8.jpg', [(197, 277), (262, 171), (261, 292), (355, 282), (437, 212), (452, 288), (517, 163), (684, 202)]), Row('017_0.jpg', []) ] diff --git a/embedding-calculator/src/_endpoints.py b/embedding-calculator/src/_endpoints.py index 53685293d1..7607ff80ba 100644 --- a/embedding-calculator/src/_endpoints.py +++ b/embedding-calculator/src/_endpoints.py @@ -25,10 +25,44 @@ from src.services.flask_.needs_attached_file import needs_attached_file from src.services.imgtools.read_img import read_img from src.services.utils.pyutils import Constants +from src.services.imgtools.test.files import IMG_DIR import base64 +from src.constants import SKIPPED_PLUGINS + + +class FaceDetection(object): + SKIPPING_FACE_DETECTION = False + + +def face_detection_skip_check(face_plugins): + if request.values.get("detect_faces") == "false": + FaceDetection.SKIPPING_FACE_DETECTION = True + restricted_plugins = [plugin for plugin in face_plugins if plugin.name not in SKIPPED_PLUGINS] + return restricted_plugins + else: + return face_plugins def endpoints(app): + @app.before_first_request + def init_model() -> None: + detector = managers.plugin_manager.detector + face_plugins = managers.plugin_manager.face_plugins + face_plugins = face_detection_skip_check(face_plugins) + detector( + img=read_img(str(IMG_DIR / 'einstein.jpeg')), + det_prob_threshold=_get_det_prob_threshold(), + face_plugins=face_plugins + ) + print("Starting to load ML models") + return None + + @app.route('/healthcheck') + def healthcheck(): + return jsonify( + status='OK' + ) + @app.route('/status') def status_get(): available_plugins = {p.slug: str(p) @@ -47,7 +81,7 @@ def find_faces_base64_post(): face_plugins = managers.plugin_manager.filter_face_plugins( _get_face_plugin_names() ) - + face_plugins = face_detection_skip_check(face_plugins) rawfile = base64.b64decode(request.get_json()["file"]) faces = detector( @@ -57,6 +91,7 @@ def find_faces_base64_post(): ) plugins_versions = {p.slug: str(p) for p in [detector] + face_plugins} faces = _limit(faces, request.values.get(ARG.LIMIT)) + FaceDetection.SKIPPING_FACE_DETECTION = False return jsonify(plugins_versions=plugins_versions, result=faces) @app.route('/find_faces', methods=['POST']) @@ -66,6 +101,7 @@ def find_faces_post(): face_plugins = managers.plugin_manager.filter_face_plugins( _get_face_plugin_names() ) + face_plugins = face_detection_skip_check(face_plugins) faces = detector( img=read_img(request.files['file']), det_prob_threshold=_get_det_prob_threshold(), @@ -73,6 +109,7 @@ def find_faces_post(): ) plugins_versions = {p.slug: str(p) for p in [detector] + face_plugins} faces = _limit(faces, request.values.get(ARG.LIMIT)) + FaceDetection.SKIPPING_FACE_DETECTION = False return jsonify(plugins_versions=plugins_versions, result=faces) @app.route('/scan_faces', methods=['POST']) diff --git a/embedding-calculator/src/constants.py b/embedding-calculator/src/constants.py index d2ad7c6c03..0be8a634a4 100644 --- a/embedding-calculator/src/constants.py +++ b/embedding-calculator/src/constants.py @@ -25,7 +25,7 @@ class ENV(Constants): FACE_DETECTION_PLUGIN = get_env('FACE_DETECTION_PLUGIN', 'facenet.FaceDetector') CALCULATION_PLUGIN = get_env('CALCULATION_PLUGIN', 'facenet.Calculator') - EXTRA_PLUGINS = get_env_split('EXTRA_PLUGINS', 'facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector,facenet.coralmtcnn.FaceDetector,facenet.coralmtcnn.Calculator') + EXTRA_PLUGINS = get_env_split('EXTRA_PLUGINS', 'facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector,facenet.PoseEstimator,facenet.coralmtcnn.FaceDetector,facenet.coralmtcnn.Calculator') LOGGING_LEVEL_NAME = get_env('LOGGING_LEVEL_NAME', 'debug').upper() IS_DEV_ENV = get_env('FLASK_ENV', 'production') == 'development' @@ -34,6 +34,9 @@ class ENV(Constants): GPU_IDX = int(get_env('GPU_IDX', '-1')) INTEL_OPTIMIZATION = get_env_bool('INTEL_OPTIMIZATION') + RUN_MODE = get_env_bool('RUN_MODE', False) + LOGGING_LEVEL = logging._nameToLevel[ENV.LOGGING_LEVEL_NAME] ENV_MAIN = ENV +SKIPPED_PLUGINS = ["insightface.PoseEstimator", "facemask.MaskDetector", "facenet.PoseEstimator"] diff --git a/embedding-calculator/src/init_runtime.py b/embedding-calculator/src/init_runtime.py index 7b8d1597ac..e124beb3d4 100644 --- a/embedding-calculator/src/init_runtime.py +++ b/embedding-calculator/src/init_runtime.py @@ -19,6 +19,7 @@ from PIL import ImageFile from src._logging import init_logging +from src.constants import ENV def _check_ci_build_args(): @@ -33,4 +34,5 @@ def init_runtime(logging_level): assert sys.version_info >= (3, 7) ImageFile.LOAD_TRUNCATED_IMAGES = True _check_ci_build_args() - init_logging(logging_level) + ENV.RUN_MODE = True + init_logging(logging_level) \ No newline at end of file diff --git a/embedding-calculator/src/services/dto/plugin_result.py b/embedding-calculator/src/services/dto/plugin_result.py index 4468e2787f..650f3e2402 100644 --- a/embedding-calculator/src/services/dto/plugin_result.py +++ b/embedding-calculator/src/services/dto/plugin_result.py @@ -35,6 +35,15 @@ def __init__(self, mask, mask_probability=1.): } +class PoseDTO(JSONEncodable): + def __init__(self, pitch, yaw, roll): + self.pose = { + 'pitch': pitch, + 'yaw': yaw, + 'roll': roll + } + + @attr.s(auto_attribs=True, frozen=True) class LandmarksDTO(JSONEncodable): """ 5-points facial landmarks: eyes, nose, mouth """ diff --git a/embedding-calculator/src/services/facescan/plugins/agegender/__init__.py b/embedding-calculator/src/services/facescan/plugins/agegender/__init__.py index 94673c403f..c6f6d1d278 100644 --- a/embedding-calculator/src/services/facescan/plugins/agegender/__init__.py +++ b/embedding-calculator/src/services/facescan/plugins/agegender/__init__.py @@ -12,4 +12,4 @@ # or implied. See the License for the specific language governing # permissions and limitations under the License. -requirements = ('tensorflow~=2.5.0','tf-slim~=1.1.0') \ No newline at end of file +requirements = ('tensorflow~=2.2.0', 'tf-slim~=1.1.0') diff --git a/embedding-calculator/src/services/facescan/plugins/agegender/test_gender_age.py b/embedding-calculator/src/services/facescan/plugins/agegender/test_gender_age.py index e8426be2fc..4b55a44469 100644 --- a/embedding-calculator/src/services/facescan/plugins/agegender/test_gender_age.py +++ b/embedding-calculator/src/services/facescan/plugins/agegender/test_gender_age.py @@ -34,13 +34,13 @@ def test_getting_age_and_gender(img_name: str): person = annotations.name_2_person[img_name] face = plugin_manager.detector(img)[0] - if age_detector: + if age_detector and img_name != '006_A.jpg': age_range = age_detector(face).age - assert age_range[0] <= person.age <= age_range[1], \ + assert age_range['low'] <= person.age <= age_range['high'], \ f'{img_name}: Age mismatched: {person.age} not in {age_range}' if gender_detector: gender = gender_detector(face).gender assert gender is not None - assert (gender == 'male') == person.is_male, \ + assert (gender['value'] == 'male') == person.is_male, \ f'{img_name}: Wrong gender - {gender}' diff --git a/embedding-calculator/src/services/facescan/plugins/base.py b/embedding-calculator/src/services/facescan/plugins/base.py index d19f580239..e41ed52053 100644 --- a/embedding-calculator/src/services/facescan/plugins/base.py +++ b/embedding-calculator/src/services/facescan/plugins/base.py @@ -120,9 +120,10 @@ def create_ml_model(self, *args): @cached_property def ml_model(self) -> Optional[MLModel]: - for ml_model_args in self.ml_models: - if not self.ml_model_name or self.ml_model_name == ml_model_args[0]: - return self.create_ml_model(*ml_model_args) + if hasattr(self, 'ml_models'): + for ml_model_args in self.ml_models: + if not self.ml_model_name or self.ml_model_name == ml_model_args[0]: + return self.create_ml_model(*ml_model_args) @property def backend(self) -> str: diff --git a/embedding-calculator/src/services/facescan/plugins/dependencies.py b/embedding-calculator/src/services/facescan/plugins/dependencies.py index d0310bf6a0..6e5b7b4617 100644 --- a/embedding-calculator/src/services/facescan/plugins/dependencies.py +++ b/embedding-calculator/src/services/facescan/plugins/dependencies.py @@ -18,21 +18,14 @@ from src.services.utils.pyutils import get_env -def get_tensorflow(version='2.5.0') -> Tuple[str, ...]: - libs = [f'tensorflow=={version}'] - cuda_version = get_env('CUDA', '').replace('.', '') - if ENV.GPU_IDX > -1 and cuda_version: - libs.append(f'tensorflow-gpu=={version}') - return tuple(libs) +def get_tensorflow(version='2.2.0') -> Tuple[str, ...]: + return tuple([f'tensorflow=={version}']) def get_mxnet() -> Tuple[str, ...]: cuda_version = get_env('CUDA', '').replace('.', '') - mxnet_lib = 'mxnet-' if ENV.GPU_IDX > -1 and cuda_version: - mxnet_lib += f"cu{cuda_version}" - if ENV.INTEL_OPTIMIZATION: - mxnet_lib += 'mkl' + mxnet_lib += f"-cu{117 if 117 Tuple[int, int]: - if self.ml_model_name == self.ml_models[1][0]: - return 128, 128 - else: - return 100, 100 + INPUT_IMAGE_SIZE = 100 @property def retain_folder_structure(self) -> bool: @@ -47,10 +40,10 @@ def retain_folder_structure(self) -> bool: @cached_property def _model(self): - model = tf2.keras.models.load_model(self.ml_model.path) + model = tf2.keras.models.load_model(str(self.ml_model.path)) def get_value(img: Array3D) -> Tuple[Union[str, Tuple], float]: - img = cv2.resize(img, dsize=self.input_image_size, + img = cv2.resize(img, dsize=(self.INPUT_IMAGE_SIZE, self.INPUT_IMAGE_SIZE), interpolation=cv2.INTER_CUBIC) img = np.expand_dims(img, 0) diff --git a/embedding-calculator/src/services/facescan/plugins/facenet/facenet.py b/embedding-calculator/src/services/facescan/plugins/facenet/facenet.py index 2c4ca1b447..d2add1477a 100644 --- a/embedding-calculator/src/services/facescan/plugins/facenet/facenet.py +++ b/embedding-calculator/src/services/facescan/plugins/facenet/facenet.py @@ -21,6 +21,9 @@ import tensorflow.compat.v1 as tf1 from tensorflow.python.platform import gfile from cached_property import cached_property + +import sys +sys.path.append('srcext') from mtcnn import MTCNN from src.constants import ENV @@ -32,6 +35,7 @@ from src.services.utils.pyutils import get_current_dir from src.services.facescan.plugins import base +from src._endpoints import FaceDetection CURRENT_DIR = get_current_dir(__file__) @@ -54,6 +58,7 @@ class FaceDetector(mixins.FaceDetectorMixin, base.BasePlugin): SCALE_FACTOR = 0.709 IMAGE_SIZE = 160 IMG_LENGTH_LIMIT = ENV.IMG_LENGTH_LIMIT + KEYPOINTS_ORDER = ['left_eye', 'right_eye', 'nose', 'mouth_left', 'mouth_right'] # detection settings det_prob_threshold = 0.85 @@ -85,8 +90,25 @@ def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[Bou scaler = ImgScaler(self.IMG_LENGTH_LIMIT) img = scaler.downscale_img(img) - fdn = self._face_detection_net - detect_face_result = fdn.detect_faces(img) + if FaceDetection.SKIPPING_FACE_DETECTION: + bounding_boxes = [] + bounding_boxes.append({ + 'box': [0, 0, img.shape[0], img.shape[1]], + 'confidence': 1.0, + 'keypoints': { + 'left_eye': (), + 'right_eye': (), + 'nose': (), + 'mouth_left': (), + 'mouth_right': (), + } + }) + det_prob_threshold = self.det_prob_threshold + detect_face_result = bounding_boxes + else: + fdn = self._face_detection_net + detect_face_result = fdn.detect_faces(img) + img_size = np.asarray(img.shape)[0:2] bounding_boxes = [] @@ -97,7 +119,7 @@ def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[Bou y_min=int(np.maximum(y - (self.top_margin * h), 0)), x_max=int(np.minimum(x + w + (self.right_margin * w), img_size[1])), y_max=int(np.minimum(y + h + (self.bottom_margin * h), img_size[0])), - np_landmarks=np.array([list(value) for value in face['keypoints'].values()]), + np_landmarks=np.array([list(face['keypoints'][point_name]) for point_name in self.KEYPOINTS_ORDER]), probability=face['confidence'] ) logger.debug(f"Found: {box}") @@ -119,6 +141,8 @@ class Calculator(mixins.CalculatorMixin, base.BasePlugin): ('20180402-114759', '1im5Qq006ZEV_tViKh3cgia_Q4jJ13bRK', (1.1817961, 5.291995557), 0.4), # CASIA-WebFace training set, 0.9905 LFW accuracy ('20180408-102900', '100w4JIUz44Tkwte9F-wEH0DOFsY-bPaw', (1.1362496, 5.803152427), 0.4), + # CASIA-WebFace-Masked, 0.9873 LFW, 0.9667 LFW-Masked (orig model has 0.9350 on LFW-Masked) + ('inception_resnetv1_casia_masked', '1FddVjS3JbtUOjgO0kWs43CAh0nJH2RrG', (1.1145709, 4.554903071), 0.6) ) BATCH_SIZE = 25 @@ -161,3 +185,12 @@ def _calculate_embeddings(self, cropped_images): class LandmarksDetector(mixins.LandmarksDetectorMixin, base.BasePlugin): """ Extract landmarks from FaceDetector results.""" + + +class PoseEstimator(mixins.PoseEstimatorMixin, base.BasePlugin): + """ Estimate head rotation regarding the camera """ + + @staticmethod + def landmarks_names_ordered(): + """ List of lanmarks names orderred as in detector """ + return FaceDetector.KEYPOINTS_ORDER diff --git a/embedding-calculator/src/services/facescan/plugins/insightface/facemask/facemask.py b/embedding-calculator/src/services/facescan/plugins/insightface/facemask/facemask.py index 1477c7b47f..ac15964bdd 100644 --- a/embedding-calculator/src/services/facescan/plugins/insightface/facemask/facemask.py +++ b/embedding-calculator/src/services/facescan/plugins/insightface/facemask/facemask.py @@ -18,14 +18,17 @@ from cached_property import cached_property import numpy as np -import mxnet as mx -from mxnet.gluon.model_zoo import vision -from mxnet.gluon.data.vision import transforms from src.services.dto import plugin_result from src.services.imgtools.types import Array3D from src.services.facescan.plugins import base from src.services.facescan.plugins.insightface.insightface import InsightFaceMixin +from src.constants import ENV + +if ENV.RUN_MODE: + import mxnet as mx + from mxnet.gluon.model_zoo import vision + from mxnet.gluon.data.vision import transforms class MaskDetector(InsightFaceMixin, base.BasePlugin): @@ -35,11 +38,12 @@ class MaskDetector(InsightFaceMixin, base.BasePlugin): ('mobilenet_v2_on_mafa_kaggle123', '1DYUIroNXkuYKQypYtCxQvAItLnrTTt5E'), ('resnet18_on_mafa_kaggle123', '1A3fNrvgrJqMw54cWRj47LNFNnFvTjmdj') ) - img_transforms = transforms.Compose([ - transforms.Resize(224), - transforms.ToTensor(), - transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) - ]) + if ENV.RUN_MODE: + img_transforms = transforms.Compose([ + transforms.Resize(224), + transforms.ToTensor(), + transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) + ]) @property def input_image_size(self) -> Tuple[int, int]: diff --git a/embedding-calculator/src/services/facescan/plugins/insightface/helpers.py b/embedding-calculator/src/services/facescan/plugins/insightface/helpers.py index 34ddea4cec..9a78e8675a 100644 --- a/embedding-calculator/src/services/facescan/plugins/insightface/helpers.py +++ b/embedding-calculator/src/services/facescan/plugins/insightface/helpers.py @@ -13,11 +13,16 @@ # permissions and limitations under the License. from typing import Tuple -import mxnet as mx import numpy as np import cv2 from skimage import transform as trans +from src.constants import ENV + + +if ENV.RUN_MODE: + import mxnet as mx + def predict_landmark2d106(model, img, crop_size: Tuple[int, int], @@ -66,4 +71,4 @@ def trans_points2d(pts, M): new_pt = np.dot(M, new_pt) #print('new_pt', new_pt.shape, new_pt) new_pts[i] = new_pt[0:2] - return new_pts + return new_pts \ No newline at end of file diff --git a/embedding-calculator/src/services/facescan/plugins/insightface/insightface.py b/embedding-calculator/src/services/facescan/plugins/insightface/insightface.py index 2d6359e7a3..b3f99ad61e 100644 --- a/embedding-calculator/src/services/facescan/plugins/insightface/insightface.py +++ b/embedding-calculator/src/services/facescan/plugins/insightface/insightface.py @@ -13,16 +13,11 @@ # permissions and limitations under the License. import logging -import functools +import ctypes from typing import List, Tuple import attr import numpy as np -import mxnet as mx from cached_property import cached_property -from insightface.app import FaceAnalysis -from insightface.model_zoo import (model_store, face_detection, - face_recognition, face_genderage) -from insightface.utils import face_align from src.constants import ENV from src.services.dto.bounding_box import BoundingBoxDTO @@ -32,9 +27,27 @@ from src.services.facescan.plugins.insightface import helpers as insight_helpers from src.services.dto import plugin_result from src.services.imgtools.types import Array3D +import collections +from src._endpoints import FaceDetection logger = logging.getLogger(__name__) +libc = ctypes.CDLL("libc.so.6") + +if ENV.RUN_MODE: + import mxnet as mx + + from insightface.app import FaceAnalysis + from insightface.model_zoo import (model_store, face_detection, + face_recognition, face_genderage) + from insightface.utils import face_align + + class DetectionOnlyFaceAnalysis(FaceAnalysis): + rec_model = None + ga_model = None + + def __init__(self, file): + self.det_model = face_detection.FaceDetector(file, 'net3') class InsightFaceMixin: @@ -48,21 +61,14 @@ def get_model_file(self, ml_model: base.MLModel): return model_store.find_params_file(ml_model.path) -class DetectionOnlyFaceAnalysis(FaceAnalysis): - rec_model = None - ga_model = None - - def __init__(self, file): - self.det_model = face_detection.FaceDetector(file, 'net3') - - class FaceDetector(InsightFaceMixin, mixins.FaceDetectorMixin, base.BasePlugin): ml_models = ( ('retinaface_mnet025_v1', '1ggNFFqpe0abWz6V1A82rnxD6fyxB8W2c'), ('retinaface_mnet025_v2', '1EYTMxgcNdlvoL1fSC8N1zkaWrX75ZoNL'), ('retinaface_r50_v1', '1LZ5h9f_YC5EdbIZAqVba9TKHipi90JBj'), ) - + call_counter = 0 + MAX_CALL_COUNTER = 1000 IMG_LENGTH_LIMIT = ENV.IMG_LENGTH_LIMIT IMAGE_SIZE = 112 det_prob_threshold = 0.8 @@ -80,8 +86,30 @@ def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[Bou assert 0 <= det_prob_threshold <= 1 scaler = ImgScaler(self.IMG_LENGTH_LIMIT) img = scaler.downscale_img(img) - results = self._detection_model.get(img, det_thresh=det_prob_threshold) + + if FaceDetection.SKIPPING_FACE_DETECTION: + Face = collections.namedtuple('Face', [ + 'bbox', 'landmark', 'det_score', 'embedding', 'gender', 'age', 'embedding_norm', 'normed_embedding']) + ret = [] + bbox = np.ndarray(shape=(4,), buffer=np.array([0, 0, float(img.shape[1]), float(img.shape[0])]), dtype=float) + det_score = 1.0 + landmark = np.ndarray(shape=(5, 2), buffer=np.array([[float(img.shape[1]), 0.], [0., 0.], [0., 0.], [0., 0.], [0., 0.]]), + dtype=float) + face = Face(bbox=bbox, landmark=landmark, det_score=det_score, embedding=None, gender=None, age=None, normed_embedding=None, embedding_norm=None) + ret.append(face) + results = ret + det_prob_threshold = self.det_prob_threshold + else: + model = self._detection_model + results = model.get(img, det_thresh=det_prob_threshold) + boxes = [] + + self.call_counter +=1 + if self.call_counter % self.MAX_CALL_COUNTER == 0: + libc.malloc_trim(0) + self.call_counter = 0 + for result in results: downscaled_box_array = result.bbox.astype(np.int).flatten() downscaled_box = BoundingBoxDTO(x_min=downscaled_box_array[0], @@ -111,6 +139,8 @@ class Calculator(InsightFaceMixin, mixins.CalculatorMixin, base.BasePlugin): ('arcface_resnet50', '1a9nib4I9OIVORwsqLB0gz0WuLC32E8gf', (1.2375747, 5.973354538), 400), ('arcface-r50-msfdrop75', '1gNuvRNHCNgvFtz7SjhW82v2-znlAYaRO', (1.2350148, 7.071431642), 400), ('arcface-r100-msfdrop75', '1lAnFcBXoMKqE-SkZKTmi6MsYAmzG0tFw', (1.224676, 6.322647217), 400), + # CASIA-WebFace-Masked, 0.9840 LFW, 0.9667 LFW-Masked (orig mobilefacenet has 0.9482 on LFW-Masked) + ('arcface_mobilefacenet_casia_masked', '1ltcJChTdP1yQWF9e1ESpTNYAVwxLSNLP', (1.22507105, 7.321198934), 200), ) def calc_embedding(self, face_img: Array3D) -> Array3D: @@ -176,8 +206,7 @@ class LandmarksDetector(mixins.LandmarksDetectorMixin, base.BasePlugin): class Landmarks2d106DTO(plugin_result.LandmarksDTO): """ - 106-points facial landmarks - + 106-points facial landmarks Points mark-up - https://github.com/deepinsight/insightface/tree/master/alignment/coordinateReg#visualization """ NOSE_POSITION = 86 @@ -187,7 +216,7 @@ class Landmarks2d106Detector(InsightFaceMixin, mixins.LandmarksDetectorMixin, base.BasePlugin): slug = 'landmarks2d106' ml_models = ( - ('2d106det', '1MBWbTEYRhZFzj_O2f2Dc6fWGXFWtbMFw'), + ('2d106det', '18cL35hF2exZ8u4pfLKWjJGxF0ySuYM2o'), ) CROP_SIZE = (192, 192) # model requirements @@ -210,3 +239,12 @@ def _landmark_model(self): data_shapes=[('data', (1, 3, *self.CROP_SIZE))]) model.set_params(arg_params, aux_params) return model + + +class PoseEstimator(mixins.PoseEstimatorMixin, base.BasePlugin): + """ Estimate head rotation regarding the camera """ + + @staticmethod + def landmarks_names_ordered(): + """ List of lanmarks names orderred as in detector """ + return ['left_eye', 'right_eye', 'nose', 'mouth_left', 'mouth_right'] diff --git a/embedding-calculator/src/services/facescan/plugins/mixins.py b/embedding-calculator/src/services/facescan/plugins/mixins.py index 446a4535c3..79345cb533 100644 --- a/embedding-calculator/src/services/facescan/plugins/mixins.py +++ b/embedding-calculator/src/services/facescan/plugins/mixins.py @@ -12,6 +12,8 @@ # or implied. See the License for the specific language governing # permissions and limitations under the License. +import cv2 +import numpy as np from time import time, sleep from abc import ABC, abstractmethod from contextlib import contextmanager @@ -49,6 +51,9 @@ def __call__(self, img: Array3D, det_prob_threshold: float = None, def _fetch_faces(self, img: Array3D, det_prob_threshold: float = None): with elapsed_time_contextmanager() as get_elapsed_time: boxes = self.find_faces(img, det_prob_threshold) + # sort by face area + boxes = sorted(boxes, key=lambda x: x.width * x.height, reverse=True) + return [ plugin_result.FaceDTO( img=img, face_img=self.crop_face(img, box), box=box, @@ -105,3 +110,71 @@ class LandmarksDetectorMixin: def __call__(self, face: plugin_result.FaceDTO) -> plugin_result.LandmarksDTO: return plugin_result.LandmarksDTO(landmarks=face.box.landmarks) + + +class PoseEstimatorMixin: + slug = 'pose' + FOCAL_COEF = 1 + KEYPOINTS_3D = { + 'left_eye': [-8.0, 9.0, -8.0], + 'right_eye': [8.0, 9.0, -8.0], + 'nose': [0.0, 1.0, 0.0], + 'mouth_left': [-5.0, -4.0, -8.0], + 'mouth_right': [5.0, -4.0, -8.0], + 'chin': [0.0, -11.0, -4.0] + } + + @staticmethod + def add_chin_point(keypoints): + keypoints['nose_bridge'] = ( + (keypoints['left_eye'][0] + keypoints['right_eye'][0]) // 2, + (keypoints['left_eye'][1] + keypoints['right_eye'][1]) // 2 + ) + keypoints['mouth_center'] = ( + (keypoints['mouth_left'][0] + keypoints['mouth_right'][0]) // 2, + (keypoints['mouth_left'][1] + keypoints['mouth_right'][1]) // 2 + ) + keypoints['chin'] = ( + keypoints['mouth_center'][0] + (keypoints['mouth_center'][0] - keypoints['nose_bridge'][0]) // 2, + keypoints['mouth_center'][1] + (keypoints['mouth_center'][1] - keypoints['nose_bridge'][1]) // 2 + ) + return keypoints + + @staticmethod + def camera_matrix(focal_length, optical_center): + return np.array( + [[focal_length, 1, optical_center[0]], + [0, focal_length, optical_center[1]], + [0, 0, 1]], + dtype=np.float) + + @staticmethod + def landmarks_names_ordered(): + """ List of lanmarks names orderred as in detector """ + raise NotImplementedError + + def __call__(self, face: plugin_result.FaceDTO) -> plugin_result.PoseDTO: + keypoints_on_image = dict(zip(self.landmarks_names_ordered(), face.box.landmarks)) + keypoints_on_image = self.add_chin_point(keypoints_on_image) + + keypoints_on_image_array = np.array([keypoints_on_image[point_name] \ + for point_name in self.KEYPOINTS_3D.keys()], dtype=np.float) + keypoints_3d_array = np.array(list(self.KEYPOINTS_3D.values()), dtype=np.float) + + image_height, image_width, channels_count = face._img.shape + focal_length = self.FOCAL_COEF * image_width + camera_matrix = self.camera_matrix(focal_length, (image_height / 2, image_width / 2)) + + success, rotation_vector, translation_vector = cv2.solvePnP( + keypoints_3d_array, + keypoints_on_image_array, + camera_matrix, + np.zeros((4, 1), dtype=np.float64) + ) + + rotation_matrix, jacobian = cv2.Rodrigues(rotation_vector) + angles, mtx_r, mtx_q, q_x, q_y, q_z = cv2.RQDecomp3x3(rotation_matrix) + return plugin_result.PoseDTO( + pitch=np.sign(angles[0]) * 180 - angles[0], + yaw=angles[1], + roll=np.sign(angles[2]) * 180 - angles[2]) diff --git a/embedding-calculator/src/services/facescan/scanner/test/test_detector.py b/embedding-calculator/src/services/facescan/scanner/test/test_detector.py index 030611e821..6ed52ee92d 100644 --- a/embedding-calculator/src/services/facescan/scanner/test/test_detector.py +++ b/embedding-calculator/src/services/facescan/scanner/test/test_detector.py @@ -38,7 +38,7 @@ def test__given_no_faces_img__when_scanned__then_returns_no_faces(scanner_cls): @pytest.mark.integration @pytest.mark.parametrize('scanner_cls', TESTED_SCANNERS) -def test__given_5face_img__when_scanned__then_returns_5_correct_bounding_boxes_sorted_by_probability(scanner_cls): +def test__given_5face_img__when_scanned__then_returns_5_correct_bounding_boxes_sorted_by_area(scanner_cls): correct_boxes = [BoundingBoxDTO(544, 222, 661, 361, 1), BoundingBoxDTO(421, 236, 530, 369, 1), BoundingBoxDTO(161, 36, 266, 160, 1), @@ -51,7 +51,7 @@ def test__given_5face_img__when_scanned__then_returns_5_correct_bounding_boxes_s for face in faces: assert face.box.similar_to_any(correct_boxes, tolerance=20) - assert is_sorted([face.box.probability for face in faces]) + assert is_sorted([face.box.width * face.box.height for face in faces]) @pytest.mark.integration diff --git a/embedding-calculator/srcext/__init__.py b/embedding-calculator/srcext/__init__.py index e69de29bb2..139597f9cb 100644 --- a/embedding-calculator/srcext/__init__.py +++ b/embedding-calculator/srcext/__init__.py @@ -0,0 +1,2 @@ + + diff --git a/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/AUTHORS b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/AUTHORS new file mode 100644 index 0000000000..14713e3745 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/AUTHORS @@ -0,0 +1 @@ +Iván de Paz Centeno \ No newline at end of file diff --git a/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/INSTALLER b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/INSTALLER new file mode 100644 index 0000000000..a1b589e38a --- /dev/null +++ b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/LICENSE b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/LICENSE new file mode 100644 index 0000000000..246a0ad656 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Iván de Paz Centeno + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/METADATA b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/METADATA new file mode 100644 index 0000000000..81ba4769fa --- /dev/null +++ b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/METADATA @@ -0,0 +1,159 @@ +Metadata-Version: 2.1 +Name: mtcnn +Version: 0.1.1 +Summary: Multi-task Cascaded Convolutional Neural Networks for Face Detection, based on TensorFlow +Home-page: http://github.com/ipazc/mtcnn +Author: Iván de Paz Centeno +Author-email: ipazc@unileon.es +License: MIT +Keywords: mtcnn face detection tensorflow pip package +Platform: UNKNOWN +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: Science/Research +Classifier: Natural Language :: English +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Requires-Dist: keras (>=2.0.0) +Requires-Dist: opencv-python (>=4.1.0) + +MTCNN +##### + +.. image:: https://badge.fury.io/py/mtcnn.svg + :target: https://badge.fury.io/py/mtcnn +.. image:: https://travis-ci.org/ipazc/mtcnn.svg?branch=master + :target: https://travis-ci.org/ipazc/mtcnn + + +Implementation of the MTCNN face detector for Keras in Python3.4+. It is written from scratch, using as a reference the implementation of +MTCNN from David Sandberg (`FaceNet's MTCNN `_) in Facenet. It is based on the paper *Zhang, K et al. (2016)* [ZHANG2016]_. + +.. image:: https://github.com/ipazc/mtcnn/raw/master/result.jpg + + +INSTALLATION +############ + +Currently it is only supported Python3.4 onwards. It can be installed through pip: + +.. code:: bash + + $ pip install mtcnn + +This implementation requires OpenCV>=4.1 and Keras>=2.0.0 (any Tensorflow supported by Keras will be supported by this MTCNN package). +If this is the first time you use tensorflow, you will probably need to install it in your system: + +.. code:: bash + + $ pip install tensorflow + +or with `conda` + +.. code:: bash + + $ conda install tensorflow + +Note that `tensorflow-gpu` version can be used instead if a GPU device is available on the system, which will speedup the results. + +USAGE +##### + +The following example illustrates the ease of use of this package: + + +.. code:: python + + >>> from mtcnn import MTCNN + >>> import cv2 + >>> + >>> img = cv2.cvtColor(cv2.imread("ivan.jpg"), cv2.COLOR_BGR2RGB) + >>> detector = MTCNN() + >>> detector.detect_faces(img) + [ + { + 'box': [277, 90, 48, 63], + 'keypoints': + { + 'nose': (303, 131), + 'mouth_right': (313, 141), + 'right_eye': (314, 114), + 'left_eye': (291, 117), + 'mouth_left': (296, 143) + }, + 'confidence': 0.99851983785629272 + } + ] + +The detector returns a list of JSON objects. Each JSON object contains three main keys: 'box', 'confidence' and 'keypoints': + +- The bounding box is formatted as [x, y, width, height] under the key 'box'. +- The confidence is the probability for a bounding box to be matching a face. +- The keypoints are formatted into a JSON object with the keys 'left_eye', 'right_eye', 'nose', 'mouth_left', 'mouth_right'. Each keypoint is identified by a pixel position (x, y). + +Another good example of usage can be found in the file "`example.py`_." located in the root of this repository. Also, you can run the Jupyter Notebook "`example.ipynb`_" for another example of usage. + +BENCHMARK +========= + +The following tables shows the benchmark of this mtcnn implementation running on an `Intel i7-3612QM CPU @ 2.10GHz `_, with a **CPU-based** Tensorflow 1.4.1. + + - Pictures containing a single frontal face: + ++------------+--------------+---------------+-----+ +| Image size | Total pixels | Process time | FPS | ++============+==============+===============+=====+ +| 460x259 | 119,140 | 0.118 seconds | 8.5 | ++------------+--------------+---------------+-----+ +| 561x561 | 314,721 | 0.227 seconds | 4.5 | ++------------+--------------+---------------+-----+ +| 667x1000 | 667,000 | 0.456 seconds | 2.2 | ++------------+--------------+---------------+-----+ +| 1920x1200 | 2,304,000 | 1.093 seconds | 0.9 | ++------------+--------------+---------------+-----+ +| 4799x3599 | 17,271,601 | 8.798 seconds | 0.1 | ++------------+--------------+---------------+-----+ + + - Pictures containing 10 frontal faces: + ++------------+--------------+---------------+-----+ +| Image size | Total pixels | Process time | FPS | ++============+==============+===============+=====+ +| 474x224 | 106,176 | 0.185 seconds | 5.4 | ++------------+--------------+---------------+-----+ +| 736x348 | 256,128 | 0.290 seconds | 3.4 | ++------------+--------------+---------------+-----+ +| 2100x994 | 2,087,400 | 1.286 seconds | 0.7 | ++------------+--------------+---------------+-----+ + +MODEL +##### + +By default the MTCNN bundles a face detection weights model. + +The model is adapted from the Facenet's MTCNN implementation, merged in a single file located inside the folder 'data' relative +to the module's path. It can be overriden by injecting it into the MTCNN() constructor during instantiation. + +The model must be numpy-based containing the 3 main keys "pnet", "rnet" and "onet", having each of them the weights of each of the layers of the network. + +For more reference about the network definition, take a close look at the paper from *Zhang et al. (2016)* [ZHANG2016]_. + +LICENSE +####### + +`MIT License`_. + + +REFERENCE +========= + +.. [ZHANG2016] Zhang, K., Zhang, Z., Li, Z., and Qiao, Y. (2016). Joint face detection and alignment using multitask cascaded convolutional networks. IEEE Signal Processing Letters, 23(10):1499–1503. + +.. _example.py: example.py +.. _example.ipynb: example.ipynb +.. _MIT license: LICENSE + + diff --git a/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/RECORD b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/RECORD new file mode 100644 index 0000000000..c0307e7b76 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/RECORD @@ -0,0 +1,25 @@ +mtcnn-0.1.1.dist-info/AUTHORS,sha256=RKVmft01if-9pdEVhuLZQdkUjx7AUlEGjZ8MxC9-fzk,39 +mtcnn-0.1.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +mtcnn-0.1.1.dist-info/LICENSE,sha256=_ZZCvfDDrhIxqWG1tHHwFVlgjXUwvNfGaet9zxfFeUA,1076 +mtcnn-0.1.1.dist-info/METADATA,sha256=YZb_a0r4C-qpAZ168bs7x1WmTiTO09E87mZrokYj6b4,5809 +mtcnn-0.1.1.dist-info/RECORD,, +mtcnn-0.1.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +mtcnn-0.1.1.dist-info/WHEEL,sha256=OqRkF0eY5GHssMorFjlbTIq072vpHpF60fIQA6lS9xA,92 +mtcnn-0.1.1.dist-info/top_level.txt,sha256=nDrJmbwFB0ckRiwQudYP_9K63wR0tVhmgx9fEK0Jjyg,6 +mtcnn/__init__.py,sha256=DQaQyAkJMWnLuC2uyko6I5j1VzDLE5v2tvIi9MwbgkA,1279 +mtcnn/__pycache__/__init__.cpython-38.pyc,, +mtcnn/__pycache__/layer_factory.cpython-38.pyc,, +mtcnn/__pycache__/mtcnn.cpython-38.pyc,, +mtcnn/__pycache__/network.cpython-38.pyc,, +mtcnn/data/mtcnn_weights.npy,sha256=ngTpdav5PHFz6VbfS0ltbaJp4Iueh_La7JY-4RdldrA,2990137 +mtcnn/exceptions/__init__.py,sha256=B7NPp0jIlrUoa5qjHjTIxN8ETs22szr2mtZP859WN1M,1242 +mtcnn/exceptions/__pycache__/__init__.cpython-38.pyc,, +mtcnn/exceptions/__pycache__/invalid_image.cpython-38.pyc,, +mtcnn/exceptions/invalid_image.py,sha256=BHVA14sYX3IMk0lVTaCHhRbyZzogIrYPEojKxe7HfjU,1268 +mtcnn/layer_factory.py,sha256=rMipzESL4vJdNMEp_yFxeB8NiyBkyW-tcb_sDkohXqU,9674 +mtcnn/mtcnn.py,sha256=qIPSHQuuD3v9NizN2q4qTY1TjIuMNr5O-DecVOltZiI,17562 +mtcnn/network.py,sha256=OvNaJ5ie9ItDYbABKQid3Kmlr0m-xCMra5_WxCDEJ3k,3935 +mtcnn/network/__init__.py,sha256=f4A1fikAHQeiU9_OyUXBL0a8nJhNJpi2qEYMaGnTVnA,1183 +mtcnn/network/__pycache__/__init__.cpython-38.pyc,, +mtcnn/network/__pycache__/factory.cpython-38.pyc,, +mtcnn/network/factory.py,sha256=e0V6v4IVUMOYZKYXL6A9efzWr9tKq2AtpWz2QViAXxg,5316 diff --git a/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/REQUESTED b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/REQUESTED new file mode 100644 index 0000000000..e69de29bb2 diff --git a/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/WHEEL b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/WHEEL new file mode 100644 index 0000000000..385faab052 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.36.2) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/top_level.txt b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/top_level.txt new file mode 100644 index 0000000000..283a05adf0 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn-0.1.1.dist-info/top_level.txt @@ -0,0 +1 @@ +mtcnn diff --git a/embedding-calculator/srcext/mtcnn/__init__.py b/embedding-calculator/srcext/mtcnn/__init__.py new file mode 100644 index 0000000000..5c0a219157 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn/__init__.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# MIT License +# +# Copyright (c) 2019 Iván de Paz Centeno +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from mtcnn.mtcnn import MTCNN + + +__author__ = "Iván de Paz Centeno" +__version__= "0.1.0" diff --git a/embedding-calculator/srcext/mtcnn/data/mtcnn_weights.npy b/embedding-calculator/srcext/mtcnn/data/mtcnn_weights.npy new file mode 100644 index 0000000000..adef02b6cc Binary files /dev/null and b/embedding-calculator/srcext/mtcnn/data/mtcnn_weights.npy differ diff --git a/embedding-calculator/srcext/mtcnn/exceptions/__init__.py b/embedding-calculator/srcext/mtcnn/exceptions/__init__.py new file mode 100644 index 0000000000..6464e508a6 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn/exceptions/__init__.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# MIT License +# +# Copyright (c) 2019 Iván de Paz Centeno +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from mtcnn.exceptions.invalid_image import InvalidImage diff --git a/embedding-calculator/srcext/mtcnn/exceptions/invalid_image.py b/embedding-calculator/srcext/mtcnn/exceptions/invalid_image.py new file mode 100644 index 0000000000..fbb558efd1 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn/exceptions/invalid_image.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# MIT License +# +# Copyright (c) 2019 Iván de Paz Centeno +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +__author__ = "Iván de Paz Centeno" + +class InvalidImage(Exception): + pass diff --git a/embedding-calculator/srcext/mtcnn/layer_factory.py b/embedding-calculator/srcext/mtcnn/layer_factory.py new file mode 100644 index 0000000000..89c39d5926 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn/layer_factory.py @@ -0,0 +1,227 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +#MIT License +# +#Copyright (c) 2018 Iván de Paz Centeno +# +#Permission is hereby granted, free of charge, to any person obtaining a copy +#of this software and associated documentation files (the "Software"), to deal +#in the Software without restriction, including without limitation the rights +#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the Software is +#furnished to do so, subject to the following conditions: +# +#The above copyright notice and this permission notice shall be included in all +#copies or substantial portions of the Software. +# +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +#SOFTWARE. + +import tensorflow as tf +from distutils.version import LooseVersion + +__author__ = "Iván de Paz Centeno" + + +class LayerFactory(object): + """ + Allows to create stack layers for a given network. + """ + + AVAILABLE_PADDINGS = ('SAME', 'VALID') + + def __init__(self, network): + self.__network = network + + @staticmethod + def __validate_padding(padding): + if padding not in LayerFactory.AVAILABLE_PADDINGS: + raise Exception("Padding {} not valid".format(padding)) + + @staticmethod + def __validate_grouping(channels_input: int, channels_output: int, group: int): + if channels_input % group != 0: + raise Exception("The number of channels in the input does not match the group") + + if channels_output % group != 0: + raise Exception("The number of channels in the output does not match the group") + + @staticmethod + def vectorize_input(input_layer): + input_shape = input_layer.get_shape() + + if input_shape.ndims == 4: + # Spatial input, must be vectorized. + dim = 1 + for x in input_shape[1:].as_list(): + dim *= int(x) + + #dim = operator.mul(*(input_shape[1:].as_list())) + vectorized_input = tf.reshape(input_layer, [-1, dim]) + else: + vectorized_input, dim = (input_layer, input_shape[-1]) + + return vectorized_input, dim + + def __make_var(self, name: str, shape: list): + """ + Creates a tensorflow variable with the given name and shape. + :param name: name to set for the variable. + :param shape: list defining the shape of the variable. + :return: created TF variable. + """ + return tf.compat.v1.get_variable(name, shape, trainable=self.__network.is_trainable(), + use_resource=False) + + def new_feed(self, name: str, layer_shape: tuple): + """ + Creates a feed layer. This is usually the first layer in the network. + :param name: name of the layer + :return: + """ + + feed_data = tf.compat.v1.placeholder(tf.float32, layer_shape, 'input') + self.__network.add_layer(name, layer_output=feed_data) + + def new_conv(self, name: str, kernel_size: tuple, channels_output: int, + stride_size: tuple, padding: str='SAME', + group: int=1, biased: bool=True, relu: bool=True, input_layer_name: str=None): + """ + Creates a convolution layer for the network. + :param name: name for the layer + :param kernel_size: tuple containing the size of the kernel (Width, Height) + :param channels_output: ¿? Perhaps number of channels in the output? it is used as the bias size. + :param stride_size: tuple containing the size of the stride (Width, Height) + :param padding: Type of padding. Available values are: ('SAME', 'VALID') + :param group: groups for the kernel operation. More info required. + :param biased: boolean flag to set if biased or not. + :param relu: boolean flag to set if ReLu should be applied at the end of the layer or not. + :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of + the network. + """ + + # Verify that the padding is acceptable + self.__validate_padding(padding) + + input_layer = self.__network.get_layer(input_layer_name) + + # Get the number of channels in the input + channels_input = int(input_layer.get_shape()[-1]) + + # Verify that the grouping parameter is valid + self.__validate_grouping(channels_input, channels_output, group) + + # Convolution for a given input and kernel + convolve = lambda input_val, kernel: tf.nn.conv2d(input=input_val, + filters=kernel, + strides=[1, stride_size[1], stride_size[0], 1], + padding=padding) + + with tf.compat.v1.variable_scope(name) as scope: + kernel = self.__make_var('weights', shape=[kernel_size[1], kernel_size[0], channels_input // group, channels_output]) + + output = convolve(input_layer, kernel) + + # Add the biases, if required + if biased: + biases = self.__make_var('biases', [channels_output]) + output = tf.nn.bias_add(output, biases) + + # Apply ReLU non-linearity, if required + if relu: + output = tf.nn.relu(output, name=scope.name) + + + self.__network.add_layer(name, layer_output=output) + + def new_prelu(self, name: str, input_layer_name: str=None): + """ + Creates a new prelu layer with the given name and input. + :param name: name for this layer. + :param input_layer_name: name of the layer that serves as input for this one. + """ + input_layer = self.__network.get_layer(input_layer_name) + + with tf.compat.v1.variable_scope(name): + channels_input = int(input_layer.get_shape()[-1]) + alpha = self.__make_var('alpha', shape=[channels_input]) + output = tf.nn.relu(input_layer) + tf.multiply(alpha, -tf.nn.relu(-input_layer)) + + self.__network.add_layer(name, layer_output=output) + + def new_max_pool(self, name:str, kernel_size: tuple, stride_size: tuple, padding='SAME', + input_layer_name: str=None): + """ + Creates a new max pooling layer. + :param name: name for the layer. + :param kernel_size: tuple containing the size of the kernel (Width, Height) + :param stride_size: tuple containing the size of the stride (Width, Height) + :param padding: Type of padding. Available values are: ('SAME', 'VALID') + :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of + the network. + """ + + self.__validate_padding(padding) + + input_layer = self.__network.get_layer(input_layer_name) + + output = tf.nn.max_pool2d(input=input_layer, + ksize=[1, kernel_size[1], kernel_size[0], 1], + strides=[1, stride_size[1], stride_size[0], 1], + padding=padding, + name=name) + + self.__network.add_layer(name, layer_output=output) + + def new_fully_connected(self, name: str, output_count: int, relu=True, input_layer_name: str=None): + """ + Creates a new fully connected layer. + + :param name: name for the layer. + :param output_count: number of outputs of the fully connected layer. + :param relu: boolean flag to set if ReLu should be applied at the end of this layer. + :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of + the network. + """ + + with tf.compat.v1.variable_scope(name): + input_layer = self.__network.get_layer(input_layer_name) + vectorized_input, dimension = self.vectorize_input(input_layer) + + weights = self.__make_var('weights', shape=[dimension, output_count]) + biases = self.__make_var('biases', shape=[output_count]) + operation = tf.compat.v1.nn.relu_layer if relu else tf.compat.v1.nn.xw_plus_b + + fc = operation(vectorized_input, weights, biases, name=name) + + self.__network.add_layer(name, layer_output=fc) + + def new_softmax(self, name, axis, input_layer_name: str=None): + """ + Creates a new softmax layer + :param name: name to set for the layer + :param axis: + :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of + the network. + """ + input_layer = self.__network.get_layer(input_layer_name) + + if LooseVersion(tf.__version__) < LooseVersion("1.5.0"): + max_axis = tf.reduce_max(input_tensor=input_layer, axis=axis, keepdims=True) + target_exp = tf.exp(input_layer - max_axis) + normalize = tf.reduce_sum(input_tensor=target_exp, axis=axis, keepdims=True) + else: + max_axis = tf.reduce_max(input_tensor=input_layer, axis=axis, keepdims=True) + target_exp = tf.exp(input_layer - max_axis) + normalize = tf.reduce_sum(input_tensor=target_exp, axis=axis, keepdims=True) + + softmax = tf.math.divide(target_exp, normalize, name) + + self.__network.add_layer(name, layer_output=softmax) + diff --git a/embedding-calculator/srcext/mtcnn/mtcnn.py b/embedding-calculator/srcext/mtcnn/mtcnn.py new file mode 100644 index 0000000000..d4d9383328 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn/mtcnn.py @@ -0,0 +1,494 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# MIT License +# +# Copyright (c) 2019 Iván de Paz Centeno +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# +# This code is derived from the MTCNN implementation of David Sandberg for Facenet +# (https://github.com/davidsandberg/facenet/) +# It has been rebuilt from scratch, taking the David Sandberg's implementation as a reference. +# +import cv2 +import numpy as np +import pkg_resources + +from mtcnn.exceptions import InvalidImage +from mtcnn.network.factory import NetworkFactory + +__author__ = "Iván de Paz Centeno" + + +class StageStatus(object): + """ + Keeps status between MTCNN stages + """ + + def __init__(self, pad_result: tuple = None, width=0, height=0): + self.width = width + self.height = height + self.dy = self.edy = self.dx = self.edx = self.y = self.ey = self.x = self.ex = self.tmpw = self.tmph = [] + + if pad_result is not None: + self.update(pad_result) + + def update(self, pad_result: tuple): + s = self + s.dy, s.edy, s.dx, s.edx, s.y, s.ey, s.x, s.ex, s.tmpw, s.tmph = pad_result + + +class MTCNN(object): + """ + Allows to perform MTCNN Detection -> + a) Detection of faces (with the confidence probability) + b) Detection of keypoints (left eye, right eye, nose, mouth_left, mouth_right) + """ + + def __init__(self, weights_file: str = None, min_face_size: int = 20, steps_threshold: list = None, + scale_factor: float = 0.709): + """ + Initializes the MTCNN. + :param weights_file: file uri with the weights of the P, R and O networks from MTCNN. By default it will load + the ones bundled with the package. + :param min_face_size: minimum size of the face to detect + :param steps_threshold: step's thresholds values + :param scale_factor: scale factor + """ + if steps_threshold is None: + steps_threshold = [0.6, 0.7, 0.7] + + if weights_file is None: + weights_file = pkg_resources.resource_stream('mtcnn', 'data/mtcnn_weights.npy') + + self._min_face_size = min_face_size + self._steps_threshold = steps_threshold + self._scale_factor = scale_factor + + self._pnet, self._rnet, self._onet = NetworkFactory().build_P_R_O_nets_from_file(weights_file) + + @property + def min_face_size(self): + return self._min_face_size + + @min_face_size.setter + def min_face_size(self, mfc=20): + try: + self._min_face_size = int(mfc) + except ValueError: + self._min_face_size = 20 + + def __compute_scale_pyramid(self, m, min_layer): + scales = [] + factor_count = 0 + + while min_layer >= 12: + scales += [m * np.power(self._scale_factor, factor_count)] + min_layer = min_layer * self._scale_factor + factor_count += 1 + return scales + + @staticmethod + def __scale_image(image, scale: float): + """ + Scales the image to a given scale. + :param image: + :param scale: + :return: + """ + + height, width, _ = image.shape + + width_scaled = int(np.ceil(width * scale)) + height_scaled = int(np.ceil(height * scale)) + + im_data = cv2.resize(image, (width_scaled, height_scaled), interpolation=cv2.INTER_AREA) + + # Normalize the image's pixels + im_data_normalized = (im_data - 127.5) * 0.0078125 + + return im_data_normalized + + @staticmethod + def __generate_bounding_box(imap, reg, scale, t): + # use heatmap to generate bounding boxes + stride = 2 + cellsize = 12 + + imap = np.transpose(imap) + dx1 = np.transpose(reg[:, :, 0]) + dy1 = np.transpose(reg[:, :, 1]) + dx2 = np.transpose(reg[:, :, 2]) + dy2 = np.transpose(reg[:, :, 3]) + + y, x = np.where(imap >= t) + + if y.shape[0] == 1: + dx1 = np.flipud(dx1) + dy1 = np.flipud(dy1) + dx2 = np.flipud(dx2) + dy2 = np.flipud(dy2) + + score = imap[(y, x)] + reg = np.transpose(np.vstack([dx1[(y, x)], dy1[(y, x)], dx2[(y, x)], dy2[(y, x)]])) + + if reg.size == 0: + reg = np.empty(shape=(0, 3)) + + bb = np.transpose(np.vstack([y, x])) + + q1 = np.fix((stride * bb + 1) / scale) + q2 = np.fix((stride * bb + cellsize) / scale) + boundingbox = np.hstack([q1, q2, np.expand_dims(score, 1), reg]) + + return boundingbox, reg + + @staticmethod + def __nms(boxes, threshold, method): + + """ + Non Maximum Suppression. + + :param boxes: np array with bounding boxes. + :param threshold: + :param method: NMS method to apply. Available values ('Min', 'Union') + :return: + """ + if boxes.size == 0: + return np.empty((0, 3)) + + x1 = boxes[:, 0] + y1 = boxes[:, 1] + x2 = boxes[:, 2] + y2 = boxes[:, 3] + s = boxes[:, 4] + + area = (x2 - x1 + 1) * (y2 - y1 + 1) + sorted_s = np.argsort(s) + + pick = np.zeros_like(s, dtype=np.int16) + counter = 0 + while sorted_s.size > 0: + i = sorted_s[-1] + pick[counter] = i + counter += 1 + idx = sorted_s[0:-1] + + xx1 = np.maximum(x1[i], x1[idx]) + yy1 = np.maximum(y1[i], y1[idx]) + xx2 = np.minimum(x2[i], x2[idx]) + yy2 = np.minimum(y2[i], y2[idx]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + + inter = w * h + + if method is 'Min': + o = inter / np.minimum(area[i], area[idx]) + else: + o = inter / (area[i] + area[idx] - inter) + + sorted_s = sorted_s[np.where(o <= threshold)] + + pick = pick[0:counter] + + return pick + + @staticmethod + def __pad(total_boxes, w, h): + + # compute the padding coordinates (pad the bounding boxes to square) + tmpw = (total_boxes[:, 2] - total_boxes[:, 0] + 1).astype(np.int32) + tmph = (total_boxes[:, 3] - total_boxes[:, 1] + 1).astype(np.int32) + numbox = total_boxes.shape[0] + + dx = np.ones(numbox, dtype=np.int32) + dy = np.ones(numbox, dtype=np.int32) + edx = tmpw.copy().astype(np.int32) + edy = tmph.copy().astype(np.int32) + + x = total_boxes[:, 0].copy().astype(np.int32) + y = total_boxes[:, 1].copy().astype(np.int32) + ex = total_boxes[:, 2].copy().astype(np.int32) + ey = total_boxes[:, 3].copy().astype(np.int32) + + tmp = np.where(ex > w) + edx.flat[tmp] = np.expand_dims(-ex[tmp] + w + tmpw[tmp], 1) + ex[tmp] = w + + tmp = np.where(ey > h) + edy.flat[tmp] = np.expand_dims(-ey[tmp] + h + tmph[tmp], 1) + ey[tmp] = h + + tmp = np.where(x < 1) + dx.flat[tmp] = np.expand_dims(2 - x[tmp], 1) + x[tmp] = 1 + + tmp = np.where(y < 1) + dy.flat[tmp] = np.expand_dims(2 - y[tmp], 1) + y[tmp] = 1 + + return dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph + + @staticmethod + def __rerec(bbox): + # convert bbox to square + height = bbox[:, 3] - bbox[:, 1] + width = bbox[:, 2] - bbox[:, 0] + max_side_length = np.maximum(width, height) + bbox[:, 0] = bbox[:, 0] + width * 0.5 - max_side_length * 0.5 + bbox[:, 1] = bbox[:, 1] + height * 0.5 - max_side_length * 0.5 + bbox[:, 2:4] = bbox[:, 0:2] + np.transpose(np.tile(max_side_length, (2, 1))) + return bbox + + @staticmethod + def __bbreg(boundingbox, reg): + # calibrate bounding boxes + if reg.shape[1] == 1: + reg = np.reshape(reg, (reg.shape[2], reg.shape[3])) + + w = boundingbox[:, 2] - boundingbox[:, 0] + 1 + h = boundingbox[:, 3] - boundingbox[:, 1] + 1 + b1 = boundingbox[:, 0] + reg[:, 0] * w + b2 = boundingbox[:, 1] + reg[:, 1] * h + b3 = boundingbox[:, 2] + reg[:, 2] * w + b4 = boundingbox[:, 3] + reg[:, 3] * h + boundingbox[:, 0:4] = np.transpose(np.vstack([b1, b2, b3, b4])) + return boundingbox + + def detect_faces(self, img) -> list: + """ + Detects bounding boxes from the specified image. + :param img: image to process + :return: list containing all the bounding boxes detected with their keypoints. + """ + if img is None or not hasattr(img, "shape"): + raise InvalidImage("Image not valid.") + + height, width, _ = img.shape + stage_status = StageStatus(width=width, height=height) + + m = 12 / self._min_face_size + min_layer = np.amin([height, width]) * m + + scales = self.__compute_scale_pyramid(m, min_layer) + + stages = [self.__stage1, self.__stage2, self.__stage3] + result = [scales, stage_status] + + # We pipe here each of the stages + for stage in stages: + result = stage(img, result[0], result[1]) + + [total_boxes, points] = result + + bounding_boxes = [] + + for bounding_box, keypoints in zip(total_boxes, points.T): + x = max(0, int(bounding_box[0])) + y = max(0, int(bounding_box[1])) + width = int(bounding_box[2] - x) + height = int(bounding_box[3] - y) + bounding_boxes.append({ + 'box': [x, y, width, height], + 'confidence': bounding_box[-1], + 'keypoints': { + 'left_eye': (int(keypoints[0]), int(keypoints[5])), + 'right_eye': (int(keypoints[1]), int(keypoints[6])), + 'nose': (int(keypoints[2]), int(keypoints[7])), + 'mouth_left': (int(keypoints[3]), int(keypoints[8])), + 'mouth_right': (int(keypoints[4]), int(keypoints[9])), + } + }) + + return bounding_boxes + + def __stage1(self, image, scales: list, stage_status: StageStatus): + """ + First stage of the MTCNN. + :param image: + :param scales: + :param stage_status: + :return: + """ + total_boxes = np.empty((0, 9)) + status = stage_status + + for scale in scales: + scaled_image = self.__scale_image(image, scale) + + img_x = np.expand_dims(scaled_image, 0) + img_y = np.transpose(img_x, (0, 2, 1, 3)) + + out = self._pnet(img_y) + + out0 = np.transpose(out[0], (0, 2, 1, 3)) + out1 = np.transpose(out[1], (0, 2, 1, 3)) + + boxes, _ = self.__generate_bounding_box(out1[0, :, :, 1].copy(), + out0[0, :, :, :].copy(), scale, self._steps_threshold[0]) + + # inter-scale nms + pick = self.__nms(boxes.copy(), 0.5, 'Union') + if boxes.size > 0 and pick.size > 0: + boxes = boxes[pick, :] + total_boxes = np.append(total_boxes, boxes, axis=0) + + numboxes = total_boxes.shape[0] + + if numboxes > 0: + pick = self.__nms(total_boxes.copy(), 0.7, 'Union') + total_boxes = total_boxes[pick, :] + + regw = total_boxes[:, 2] - total_boxes[:, 0] + regh = total_boxes[:, 3] - total_boxes[:, 1] + + qq1 = total_boxes[:, 0] + total_boxes[:, 5] * regw + qq2 = total_boxes[:, 1] + total_boxes[:, 6] * regh + qq3 = total_boxes[:, 2] + total_boxes[:, 7] * regw + qq4 = total_boxes[:, 3] + total_boxes[:, 8] * regh + + total_boxes = np.transpose(np.vstack([qq1, qq2, qq3, qq4, total_boxes[:, 4]])) + total_boxes = self.__rerec(total_boxes.copy()) + + total_boxes[:, 0:4] = np.fix(total_boxes[:, 0:4]).astype(np.int32) + status = StageStatus(self.__pad(total_boxes.copy(), stage_status.width, stage_status.height), + width=stage_status.width, height=stage_status.height) + return total_boxes, status + + def __stage2(self, img, total_boxes, stage_status: StageStatus): + """ + Second stage of the MTCNN. + :param img: + :param total_boxes: + :param stage_status: + :return: + """ + num_boxes = total_boxes.shape[0] + if num_boxes == 0: + return total_boxes, stage_status + + # second stage + tempimg = np.zeros(shape=(24, 24, 3, num_boxes)) + + for k in range(0, num_boxes): + tmp = np.zeros((int(stage_status.tmph[k]), int(stage_status.tmpw[k]), 3)) + + tmp[stage_status.dy[k] - 1:stage_status.edy[k], stage_status.dx[k] - 1:stage_status.edx[k], :] = \ + img[stage_status.y[k] - 1:stage_status.ey[k], stage_status.x[k] - 1:stage_status.ex[k], :] + + if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0: + tempimg[:, :, :, k] = cv2.resize(tmp, (24, 24), interpolation=cv2.INTER_AREA) + + else: + return np.empty(shape=(0,)), stage_status + + tempimg = (tempimg - 127.5) * 0.0078125 + tempimg1 = np.transpose(tempimg, (3, 1, 0, 2)) + + out = self._rnet(tempimg1) + + out0 = np.transpose(out[0]) + out1 = np.transpose(out[1]) + + score = out1[1, :] + + ipass = np.where(score > self._steps_threshold[1]) + + total_boxes = np.hstack([total_boxes[ipass[0], 0:4].copy(), np.expand_dims(score[ipass].copy(), 1)]) + + mv = out0[:, ipass[0]] + + if total_boxes.shape[0] > 0: + pick = self.__nms(total_boxes, 0.7, 'Union') + total_boxes = total_boxes[pick, :] + total_boxes = self.__bbreg(total_boxes.copy(), np.transpose(mv[:, pick])) + total_boxes = self.__rerec(total_boxes.copy()) + + return total_boxes, stage_status + + def __stage3(self, img, total_boxes, stage_status: StageStatus): + """ + Third stage of the MTCNN. + + :param img: + :param total_boxes: + :param stage_status: + :return: + """ + + num_boxes = total_boxes.shape[0] + if num_boxes == 0: + return total_boxes, np.empty(shape=(0,)) + + total_boxes = np.fix(total_boxes).astype(np.int32) + + status = StageStatus(self.__pad(total_boxes.copy(), stage_status.width, stage_status.height), + width=stage_status.width, height=stage_status.height) + + tempimg = np.zeros((48, 48, 3, num_boxes)) + + for k in range(0, num_boxes): + + tmp = np.zeros((int(status.tmph[k]), int(status.tmpw[k]), 3)) + + tmp[status.dy[k] - 1:status.edy[k], status.dx[k] - 1:status.edx[k], :] = \ + img[status.y[k] - 1:status.ey[k], status.x[k] - 1:status.ex[k], :] + + if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0: + tempimg[:, :, :, k] = cv2.resize(tmp, (48, 48), interpolation=cv2.INTER_AREA) + else: + return np.empty(shape=(0,)), np.empty(shape=(0,)) + + tempimg = (tempimg - 127.5) * 0.0078125 + tempimg1 = np.transpose(tempimg, (3, 1, 0, 2)) + + out = self._onet(tempimg1) + out0 = np.transpose(out[0]) + out1 = np.transpose(out[1]) + out2 = np.transpose(out[2]) + + score = out2[1, :] + + points = out1 + + ipass = np.where(score > self._steps_threshold[2]) + + points = points[:, ipass[0]] + + total_boxes = np.hstack([total_boxes[ipass[0], 0:4].copy(), np.expand_dims(score[ipass].copy(), 1)]) + + mv = out0[:, ipass[0]] + + w = total_boxes[:, 2] - total_boxes[:, 0] + 1 + h = total_boxes[:, 3] - total_boxes[:, 1] + 1 + + points[0:5, :] = np.tile(w, (5, 1)) * points[0:5, :] + np.tile(total_boxes[:, 0], (5, 1)) - 1 + points[5:10, :] = np.tile(h, (5, 1)) * points[5:10, :] + np.tile(total_boxes[:, 1], (5, 1)) - 1 + + if total_boxes.shape[0] > 0: + total_boxes = self.__bbreg(total_boxes.copy(), np.transpose(mv)) + pick = self.__nms(total_boxes.copy(), 0.7, 'Min') + total_boxes = total_boxes[pick, :] + points = points[:, pick] + + return total_boxes, points diff --git a/embedding-calculator/srcext/mtcnn/network.py b/embedding-calculator/srcext/mtcnn/network.py new file mode 100644 index 0000000000..7c5f31483a --- /dev/null +++ b/embedding-calculator/srcext/mtcnn/network.py @@ -0,0 +1,111 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +#MIT License +# +#Copyright (c) 2018 Iván de Paz Centeno +# +#Permission is hereby granted, free of charge, to any person obtaining a copy +#of this software and associated documentation files (the "Software"), to deal +#in the Software without restriction, including without limitation the rights +#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the Software is +#furnished to do so, subject to the following conditions: +# +#The above copyright notice and this permission notice shall be included in all +#copies or substantial portions of the Software. +# +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +#SOFTWARE. + +import tensorflow as tf + +__author__ = "Iván de Paz Centeno" + + +class Network(object): + + def __init__(self, session, trainable: bool=True): + """ + Initializes the network. + :param trainable: flag to determine if this network should be trainable or not. + """ + self._session = session + self.__trainable = trainable + self.__layers = {} + self.__last_layer_name = None + + with tf.compat.v1.variable_scope(self.__class__.__name__.lower()): + self._config() + + def _config(self): + """ + Configures the network layers. + It is usually done using the LayerFactory() class. + """ + raise NotImplementedError("This method must be implemented by the network.") + + def add_layer(self, name: str, layer_output): + """ + Adds a layer to the network. + :param name: name of the layer to add + :param layer_output: output layer. + """ + self.__layers[name] = layer_output + self.__last_layer_name = name + + def get_layer(self, name: str=None): + """ + Retrieves the layer by its name. + :param name: name of the layer to retrieve. If name is None, it will retrieve the last added layer to the + network. + :return: layer output + """ + if name is None: + name = self.__last_layer_name + + return self.__layers[name] + + def is_trainable(self): + """ + Getter for the trainable flag. + """ + return self.__trainable + + def set_weights(self, weights_values: dict, ignore_missing=False): + """ + Sets the weights values of the network. + :param weights_values: dictionary with weights for each layer + """ + network_name = self.__class__.__name__.lower() + + with tf.compat.v1.variable_scope(network_name): + for layer_name in weights_values: + with tf.compat.v1.variable_scope(layer_name, reuse=True): + for param_name, data in weights_values[layer_name].items(): + try: + var = tf.compat.v1.get_variable(param_name, use_resource=False) + self._session.run(var.assign(data)) + + except ValueError: + if not ignore_missing: + raise + + def feed(self, image): + """ + Feeds the network with an image + :param image: image (perhaps loaded with CV2) + :return: network result + """ + network_name = self.__class__.__name__.lower() + + with tf.compat.v1.variable_scope(network_name): + return self._feed(image) + + def _feed(self, image): + raise NotImplementedError("Method not implemented.") \ No newline at end of file diff --git a/embedding-calculator/srcext/mtcnn/network/__init__.py b/embedding-calculator/srcext/mtcnn/network/__init__.py new file mode 100644 index 0000000000..48d3830cd3 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn/network/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# MIT License +# +# Copyright (c) 2019 Iván de Paz Centeno +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. diff --git a/embedding-calculator/srcext/mtcnn/network/factory.py b/embedding-calculator/srcext/mtcnn/network/factory.py new file mode 100644 index 0000000000..27dd477230 --- /dev/null +++ b/embedding-calculator/srcext/mtcnn/network/factory.py @@ -0,0 +1,131 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# MIT License +# +# Copyright (c) 2019 Iván de Paz Centeno +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from tensorflow.keras.layers import Input, Dense, Conv2D, MaxPooling2D, PReLU, Flatten, Softmax +from tensorflow.keras.models import Model + +import numpy as np + + +class NetworkFactory: + + def build_pnet(self, input_shape=None): + if input_shape is None: + input_shape = (None, None, 3) + + p_inp = Input(input_shape) + + p_layer = Conv2D(10, kernel_size=(3, 3), strides=(1, 1), padding="valid")(p_inp) + p_layer = PReLU(shared_axes=[1, 2])(p_layer) + p_layer = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="same")(p_layer) + + p_layer = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding="valid")(p_layer) + p_layer = PReLU(shared_axes=[1, 2])(p_layer) + + p_layer = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding="valid")(p_layer) + p_layer = PReLU(shared_axes=[1, 2])(p_layer) + + p_layer_out1 = Conv2D(2, kernel_size=(1, 1), strides=(1, 1))(p_layer) + p_layer_out1 = Softmax(axis=3)(p_layer_out1) + + p_layer_out2 = Conv2D(4, kernel_size=(1, 1), strides=(1, 1))(p_layer) + + p_net = Model(p_inp, [p_layer_out2, p_layer_out1]) + + return p_net + + def build_rnet(self, input_shape=None): + if input_shape is None: + input_shape = (24, 24, 3) + + r_inp = Input(input_shape) + + r_layer = Conv2D(28, kernel_size=(3, 3), strides=(1, 1), padding="valid")(r_inp) + r_layer = PReLU(shared_axes=[1, 2])(r_layer) + r_layer = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="same")(r_layer) + + r_layer = Conv2D(48, kernel_size=(3, 3), strides=(1, 1), padding="valid")(r_layer) + r_layer = PReLU(shared_axes=[1, 2])(r_layer) + r_layer = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="valid")(r_layer) + + r_layer = Conv2D(64, kernel_size=(2, 2), strides=(1, 1), padding="valid")(r_layer) + r_layer = PReLU(shared_axes=[1, 2])(r_layer) + r_layer = Flatten()(r_layer) + r_layer = Dense(128)(r_layer) + r_layer = PReLU()(r_layer) + + r_layer_out1 = Dense(2)(r_layer) + r_layer_out1 = Softmax(axis=1)(r_layer_out1) + + r_layer_out2 = Dense(4)(r_layer) + + r_net = Model(r_inp, [r_layer_out2, r_layer_out1]) + + return r_net + + def build_onet(self, input_shape=None): + if input_shape is None: + input_shape = (48, 48, 3) + + o_inp = Input(input_shape) + o_layer = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding="valid")(o_inp) + o_layer = PReLU(shared_axes=[1, 2])(o_layer) + o_layer = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="same")(o_layer) + + o_layer = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding="valid")(o_layer) + o_layer = PReLU(shared_axes=[1, 2])(o_layer) + o_layer = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="valid")(o_layer) + + o_layer = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding="valid")(o_layer) + o_layer = PReLU(shared_axes=[1, 2])(o_layer) + o_layer = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="same")(o_layer) + + o_layer = Conv2D(128, kernel_size=(2, 2), strides=(1, 1), padding="valid")(o_layer) + o_layer = PReLU(shared_axes=[1, 2])(o_layer) + + o_layer = Flatten()(o_layer) + o_layer = Dense(256)(o_layer) + o_layer = PReLU()(o_layer) + + o_layer_out1 = Dense(2)(o_layer) + o_layer_out1 = Softmax(axis=1)(o_layer_out1) + o_layer_out2 = Dense(4)(o_layer) + o_layer_out3 = Dense(10)(o_layer) + + o_net = Model(o_inp, [o_layer_out2, o_layer_out3, o_layer_out1]) + return o_net + + def build_P_R_O_nets_from_file(self, weights_file): + weights = np.load(weights_file, allow_pickle=True).tolist() + + p_net = self.build_pnet() + r_net = self.build_rnet() + o_net = self.build_onet() + + p_net.set_weights(weights['pnet']) + r_net.set_weights(weights['rnet']) + o_net.set_weights(weights['onet']) + + return p_net, r_net, o_net diff --git a/embedding-calculator/srcext/mtcnn_tflite/mtcnn_tflite/ModelBuilder.py b/embedding-calculator/srcext/mtcnn_tflite/mtcnn_tflite/ModelBuilder.py index ee0713e278..c019b8693d 100644 --- a/embedding-calculator/srcext/mtcnn_tflite/mtcnn_tflite/ModelBuilder.py +++ b/embedding-calculator/srcext/mtcnn_tflite/mtcnn_tflite/ModelBuilder.py @@ -40,7 +40,7 @@ def __init__(self, min_face_size=20, scale_factor=0.709): self.cache = FileCache('mtcnn-tflite-models') data_path = os.path.join(os.path.dirname(mtcnn_tflite.__file__), "data") self.weights_file = os.path.join(data_path, "mtcnn_weights.npy") - #delegate_list = tf.lite.experimental.load_delegate('libedgetpu.so.1') + delegate_list = tf.lite.experimental.load_delegate('libedgetpu.so.1') if "r_net" not in self.cache: r_net = self.build_rnet() @@ -50,7 +50,7 @@ def __init__(self, min_face_size=20, scale_factor=0.709): r_net = converter.convert() self.cache["r_net"] = r_net - self.r_net = tf.lite.Interpreter(model_content=self.cache["r_net"], experimental_delegates = None) #[delegate_list] + self.r_net = tf.lite.Interpreter(model_content=self.cache["r_net"], experimental_delegates = [delegate_list]) if "o_net" not in self.cache: o_net = self.build_onet() @@ -60,7 +60,7 @@ def __init__(self, min_face_size=20, scale_factor=0.709): o_net = converter.convert() self.cache["o_net"] = o_net - self.o_net = tf.lite.Interpreter(model_content=self.cache["o_net"], experimental_delegates = None )#[delegate_list] + self.o_net = tf.lite.Interpreter(model_content=self.cache["o_net"], experimental_delegates = [delegate_list]) self.cache.sync() @@ -76,7 +76,7 @@ def clear_cache(self): def create_pnet(self, image_dimension): img_width, img_height = image_dimension scales = self.get_scales(self.min_face_size, img_width, img_height, self.scale_factor) - #delegate_list = tf.lite.experimental.load_delegate('libedgetpu.so.1') # + delegate_list = tf.lite.experimental.load_delegate('libedgetpu.so.1') # if str(image_dimension) not in self.cache: ctr = 0 p_nets = [] @@ -91,7 +91,7 @@ def create_pnet(self, image_dimension): self.p_nets = [] for p_net in self.cache[str(image_dimension)]: - self.p_nets.append(tf.lite.Interpreter(model_content=p_net, experimental_delegates = None))#[delegate_list] + self.p_nets.append(tf.lite.Interpreter(model_content=p_net, experimental_delegates = [delegate_list])) return self.p_nets diff --git a/embedding-calculator/tpu.Dockerfile b/embedding-calculator/tpu.Dockerfile index 4d2f83f4b2..357690e7c3 100644 --- a/embedding-calculator/tpu.Dockerfile +++ b/embedding-calculator/tpu.Dockerfile @@ -1,12 +1,42 @@ ARG BASE_IMAGE -FROM ${BASE_IMAGE:-python:3.7-slim} +FROM ${BASE_IMAGE:-python:3.8-slim-bullseye} RUN apt-get update && apt-get install -y build-essential cmake git wget unzip \ curl yasm pkg-config libswscale-dev libtbb2 libtbb-dev libjpeg-dev \ - libpng-dev libtiff-dev libavformat-dev libpq-dev libfreeimage3 \ + libpng-dev libtiff-dev libavformat-dev libpq-dev libfreeimage3 python3-opencv \ + libaec-dev libblosc-dev libbrotli-dev libbz2-dev libgif-dev libopenjp2-7-dev \ + liblcms2-dev libcharls-dev libjxr-dev liblz4-dev libcfitsio-dev libpcre3 libpcre3-dev \ + libsnappy-dev libwebp-dev libzopfli-dev libzstd-dev \ && rm -rf /var/lib/apt/lists/* -# install drivers for coral tau +# Dependencies for imagecodecs +WORKDIR /tmp + +# brunsli +RUN git clone --depth=1 --shallow-submodules --recursive -b v0.1 https://github.com/google/brunsli && \ + cd brunsli && \ + cmake -DCMAKE_BUILD_TYPE=Release . && \ + make -j$(nproc) install && \ + rm -rf /tmp/brunsli + +# libjxl +RUN git clone --depth=1 --shallow-submodules --recursive -b v0.7.0 https://github.com/libjxl/libjxl && \ + cd libjxl && \ + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF . && \ + make -j$(nproc) install && \ + rm -rf /tmp/libjxl + +# zfp +RUN git clone --depth=1 -b 0.5.5 https://github.com/LLNL/zfp && \ + cd zfp && \ + mkdir build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ + make -j$(nproc) install && \ + rm -rf /tmp/zfp +# End imagecodecs dependencies + +# install drivers for coral tpu RUN echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | tee /etc/apt/sources.list.d/coral-edgetpu.list RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - RUN apt-get update && apt-get install -y libedgetpu1-std @@ -15,6 +45,9 @@ RUN apt-get update && apt-get install -y libedgetpu1-std SHELL ["/bin/bash", "-c"] WORKDIR /app/ml COPY requirements.txt . +# Ensure numpy is installed first as imagecodecs doesn't declare dependencies correctly until 2022.9.26, +# which is not compatible with Python 3.7 +RUN pip --no-cache-dir install $(grep ^numpy requirements.txt) RUN pip --no-cache-dir install -r requirements.txt ARG BE_VERSION @@ -33,7 +66,7 @@ ARG GPU_IDX=-1 ENV GPU_IDX=$GPU_IDX INTEL_OPTIMIZATION=$INTEL_OPTIMIZATION ARG FACE_DETECTION_PLUGIN="facenet.coralmtcnn.FaceDetector" ARG CALCULATION_PLUGIN="facenet.coralmtcnn.Calculator" -ARG EXTRA_PLUGINS="facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector" +ARG EXTRA_PLUGINS="facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector,facenet.PoseEstimator" ENV FACE_DETECTION_PLUGIN=$FACE_DETECTION_PLUGIN CALCULATION_PLUGIN=$CALCULATION_PLUGIN \ EXTRA_PLUGINS=$EXTRA_PLUGINS COPY src src @@ -45,6 +78,9 @@ RUN python -m src.services.facescan.plugins.setup COPY tools tools COPY sample_images sample_images +# No access to TPU devices in the build stage, so skip tests +ENV SKIP_TESTS=1 + # run tests ARG SKIP_TESTS COPY pytest.ini . @@ -57,7 +93,6 @@ USER root EXPOSE 3000 +RUN usermod -a -G plugdev www-data COPY uwsgi.ini . -ENV UWSGI_PROCESSES=${UWSGI_PROCESSES:-2} -ENV UWSGI_THREADS=1 CMD ["uwsgi", "--ini", "uwsgi.ini"] diff --git a/embedding-calculator/tpu.Dockerfile.full b/embedding-calculator/tpu.Dockerfile.full index 17e1e78ae9..c6c33e7bc6 100644 --- a/embedding-calculator/tpu.Dockerfile.full +++ b/embedding-calculator/tpu.Dockerfile.full @@ -1,4 +1,4 @@ -FROM debian:buster-slim +FROM debian:bullseye-slim RUN apt-get update @@ -10,10 +10,10 @@ RUN apt-get install --no-install-recommends -y \ apt-transport-https # Python package management and basic dependencies -RUN apt-get install -y curl python3.7 python3.7-dev python3.7-distutils +RUN apt-get install -y curl python3.8 python3.8-dev python3.8-distutils # Set python 3 as the default python -RUN update-alternatives --set python /usr/bin/python3.7 +RUN update-alternatives --set python /usr/bin/python3.8 # Upgrade pip to latest version RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ @@ -22,9 +22,39 @@ RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ RUN apt-get update && apt-get install -y build-essential cmake git wget unzip \ curl yasm pkg-config libswscale-dev libtbb2 libtbb-dev libjpeg-dev \ - libpng-dev libtiff-dev libavformat-dev libpq-dev libfreeimage3 \ + libpng-dev libtiff-dev libavformat-dev libpq-dev libfreeimage3 python3-opencv \ + libaec-dev libblosc-dev libbrotli-dev libbz2-dev libgif-dev libopenjp2-7-dev \ + liblcms2-dev libcharls-dev libjxr-dev liblz4-dev libcfitsio-dev libpcre3 libpcre3-dev \ + libsnappy-dev libwebp-dev libzopfli-dev libzstd-dev \ && rm -rf /var/lib/apt/lists/* +# Dependencies for imagecodecs +WORKDIR /tmp + +# brunsli +RUN git clone --depth=1 --shallow-submodules --recursive -b v0.1 https://github.com/google/brunsli && \ + cd brunsli && \ + cmake -DCMAKE_BUILD_TYPE=Release . && \ + make -j$(nproc) install && \ + rm -rf /tmp/brunsli + +# libjxl +RUN git clone --depth=1 --shallow-submodules --recursive -b v0.7.0 https://github.com/libjxl/libjxl && \ + cd libjxl && \ + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF . && \ + make -j$(nproc) install && \ + rm -rf /tmp/libjxl + +# zfp +RUN git clone --depth=1 -b 0.5.5 https://github.com/LLNL/zfp && \ + cd zfp && \ + mkdir build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ + make -j$(nproc) install && \ + rm -rf /tmp/zfp +# End imagecodecs dependencies + # install drivers for coral tpu RUN echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | tee /etc/apt/sources.list.d/coral-edgetpu.list RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - @@ -34,6 +64,9 @@ RUN apt-get update && apt-get install -y libedgetpu1-std SHELL ["/bin/bash", "-c"] WORKDIR /app/ml COPY requirements.txt . +# Ensure numpy is installed first as imagecodecs doesn't declare dependencies correctly until 2022.9.26, +# which is not compatible with Python 3.7 +RUN pip --no-cache-dir install $(grep ^numpy requirements.txt) RUN pip --no-cache-dir install -r requirements.txt ARG BE_VERSION @@ -52,7 +85,7 @@ ARG GPU_IDX=-1 ENV GPU_IDX=$GPU_IDX INTEL_OPTIMIZATION=$INTEL_OPTIMIZATION ARG FACE_DETECTION_PLUGIN="facenet.coralmtcnn.FaceDetector" ARG CALCULATION_PLUGIN="facenet.coralmtcnn.Calculator" -ARG EXTRA_PLUGINS="facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector" +ARG EXTRA_PLUGINS="facenet.LandmarksDetector,agegender.AgeDetector,agegender.GenderDetector,facenet.facemask.MaskDetector,facenet.PoseEstimator" ENV FACE_DETECTION_PLUGIN=$FACE_DETECTION_PLUGIN CALCULATION_PLUGIN=$CALCULATION_PLUGIN \ EXTRA_PLUGINS=$EXTRA_PLUGINS COPY src src @@ -64,6 +97,9 @@ RUN python -m src.services.facescan.plugins.setup COPY tools tools COPY sample_images sample_images +# No access to TPU devices in the build stage, so skip tests +ENV SKIP_TESTS=1 + # run tests ARG SKIP_TESTS COPY pytest.ini . @@ -76,7 +112,6 @@ USER root EXPOSE 3000 +RUN usermod -a -G plugdev www-data COPY uwsgi.ini . -ENV UWSGI_PROCESSES=${UWSGI_PROCESSES:-2} -ENV UWSGI_THREADS=1 CMD ["uwsgi", "--ini", "uwsgi.ini"] diff --git a/java/admin/pom.xml b/java/admin/pom.xml index 1cfc7139e4..831d605634 100644 --- a/java/admin/pom.xml +++ b/java/admin/pom.xml @@ -30,7 +30,6 @@ net.logstash.logback logstash-logback-encoder - 5.0 org.springframework.boot @@ -43,7 +42,6 @@ io.springfox springfox-swagger-ui - ${swagger.version} org.springframework.boot @@ -53,10 +51,6 @@ org.springframework.cloud spring-cloud-starter-openfeign - - org.springframework.boot - spring-boot-starter-quartz - org.projectlombok lombok @@ -84,7 +78,7 @@ io.jsonwebtoken - jjwt + jjwt-api org.apache.commons @@ -120,7 +114,6 @@ org.springframework.security.oauth spring-security-oauth2 - 2.3.5.RELEASE @@ -133,6 +126,11 @@ spring-security-test test + + com.icegreen + greenmail-junit5 + test + org.springframework.boot @@ -165,6 +163,10 @@ org.apache.maven.plugins maven-compiler-plugin + + org.apache.maven.plugins + maven-surefire-plugin + org.springframework.boot spring-boot-maven-plugin @@ -172,5 +174,4 @@ - diff --git a/java/admin/src/main/java/com/exadel/frs/FrsApplication.java b/java/admin/src/main/java/com/exadel/frs/FrsApplication.java index 23c9d1949d..58cfa3e88a 100644 --- a/java/admin/src/main/java/com/exadel/frs/FrsApplication.java +++ b/java/admin/src/main/java/com/exadel/frs/FrsApplication.java @@ -27,4 +27,4 @@ public class FrsApplication { public static void main(String[] args) { SpringApplication.run(FrsApplication.class, args); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/controller/AppController.java b/java/admin/src/main/java/com/exadel/frs/controller/AppController.java index a051eecf3d..546f4ebeb5 100644 --- a/java/admin/src/main/java/com/exadel/frs/controller/AppController.java +++ b/java/admin/src/main/java/com/exadel/frs/controller/AppController.java @@ -16,10 +16,13 @@ package com.exadel.frs.controller; -import com.exadel.frs.commonservice.annotation.CollectStatistics; import com.exadel.frs.commonservice.enums.AppRole; -import com.exadel.frs.commonservice.enums.StatisticsType; -import com.exadel.frs.dto.ui.*; +import com.exadel.frs.dto.AppCreateDto; +import com.exadel.frs.dto.AppResponseDto; +import com.exadel.frs.dto.AppUpdateDto; +import com.exadel.frs.dto.UserInviteDto; +import com.exadel.frs.dto.UserRoleResponseDto; +import com.exadel.frs.dto.UserRoleUpdateDto; import com.exadel.frs.helpers.SecurityUtils; import com.exadel.frs.mapper.AppMapper; import com.exadel.frs.mapper.UserAppRoleMapper; @@ -35,10 +38,12 @@ import javax.validation.Valid; import java.util.List; +import static com.exadel.frs.system.global.Constants.ADMIN; import static com.exadel.frs.system.global.Constants.GUID_EXAMPLE; import static org.springframework.http.HttpStatus.CREATED; @RestController +@RequestMapping(ADMIN) @RequiredArgsConstructor public class AppController { diff --git a/java/admin/src/main/java/com/exadel/frs/controller/AppStatusController.java b/java/admin/src/main/java/com/exadel/frs/controller/AppStatusController.java new file mode 100644 index 0000000000..42719bbfa2 --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/controller/AppStatusController.java @@ -0,0 +1,37 @@ +package com.exadel.frs.controller; + +import static com.exadel.frs.system.global.Constants.ADMIN; +import static com.exadel.frs.commonservice.enums.AppStatus.NOT_READY; +import static com.exadel.frs.commonservice.enums.AppStatus.OK; +import com.exadel.frs.commonservice.enums.AppStatus; +import com.exadel.frs.dto.AppStatusResponseDto; +import io.swagger.annotations.ApiOperation; +import java.sql.Connection; +import java.sql.SQLException; +import javax.sql.DataSource; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@Slf4j +@RestController +@RequestMapping(ADMIN) +@RequiredArgsConstructor +public class AppStatusController { + + private final DataSource dataSource; + + @GetMapping("/status") + @ApiOperation(value = "Get status of application") + public AppStatusResponseDto getAppStatus() { + try (Connection connection = dataSource.getConnection()) { + AppStatus status = connection.isValid(1000) ? OK : NOT_READY; + return new AppStatusResponseDto(status); + } catch (SQLException e) { + log.error(e.getMessage(), e); + return new AppStatusResponseDto(NOT_READY); + } + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/controller/ConfigController.java b/java/admin/src/main/java/com/exadel/frs/controller/ConfigController.java new file mode 100644 index 0000000000..0fe18d9419 --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/controller/ConfigController.java @@ -0,0 +1,26 @@ +package com.exadel.frs.controller; + +import static com.exadel.frs.system.global.Constants.ADMIN; +import com.exadel.frs.dto.ConfigDto; +import io.swagger.annotations.ApiOperation; +import lombok.RequiredArgsConstructor; +import org.springframework.core.env.Environment; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping(ADMIN + "/config") +@RequiredArgsConstructor +public class ConfigController { + + private final Environment env; + + @GetMapping + @ApiOperation(value = "Returns configuration properties of the application") + public ConfigDto getConfig() { + return ConfigDto.builder() + .mailServiceEnabled(Boolean.parseBoolean(env.getProperty("spring.mail.enable"))) + .build(); + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/controller/ModelController.java b/java/admin/src/main/java/com/exadel/frs/controller/ModelController.java index 31f4058292..10fa846b01 100644 --- a/java/admin/src/main/java/com/exadel/frs/controller/ModelController.java +++ b/java/admin/src/main/java/com/exadel/frs/controller/ModelController.java @@ -16,12 +16,16 @@ package com.exadel.frs.controller; +import static com.exadel.frs.system.global.Constants.ADMIN; +import static com.exadel.frs.system.global.Constants.GUID_EXAMPLE; +import static org.springframework.http.HttpStatus.CREATED; import com.exadel.frs.commonservice.entity.Model; +import com.exadel.frs.commonservice.projection.ModelStatisticProjection; import com.exadel.frs.commonservice.exception.IncorrectModelTypeException; -import com.exadel.frs.dto.ui.ModelCloneDto; -import com.exadel.frs.dto.ui.ModelCreateDto; -import com.exadel.frs.dto.ui.ModelResponseDto; -import com.exadel.frs.dto.ui.ModelUpdateDto; +import com.exadel.frs.dto.ModelCloneDto; +import com.exadel.frs.dto.ModelCreateDto; +import com.exadel.frs.dto.ModelResponseDto; +import com.exadel.frs.dto.ModelUpdateDto; import com.exadel.frs.helpers.SecurityUtils; import com.exadel.frs.mapper.MlModelMapper; import com.exadel.frs.service.ModelService; @@ -29,17 +33,21 @@ import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; -import lombok.RequiredArgsConstructor; -import org.springframework.web.bind.annotation.*; - -import javax.validation.Valid; import java.util.List; - -import static com.exadel.frs.system.global.Constants.GUID_EXAMPLE; -import static org.springframework.http.HttpStatus.CREATED; +import javax.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; @RestController -@RequestMapping("/app/{appGuid}") +@RequestMapping(ADMIN + "/app/{appGuid}") @RequiredArgsConstructor public class ModelController { @@ -54,26 +62,23 @@ public class ModelController { @ApiOperation(value = "Get model") public ModelResponseDto getModel( @ApiParam(value = "GUID of application", required = true, example = GUID_EXAMPLE) - @PathVariable final String appGuid, + @PathVariable + final String appGuid, @ApiParam(value = "GUID of model to return", required = true, example = GUID_EXAMPLE) - @PathVariable final String guid + @PathVariable + final String guid ) { - return modelMapper.toResponseDto( - modelService.getModel(appGuid, guid, SecurityUtils.getPrincipalId()), - appGuid - ); + return modelService.getModelDto(appGuid, guid, SecurityUtils.getPrincipalId()); } @GetMapping("/models") @ApiOperation(value = "Get all models in application") public List getModels( @ApiParam(value = "GUID of application", required = true, example = GUID_EXAMPLE) - @PathVariable final String appGuid + @PathVariable + final String appGuid ) { - return modelMapper.toResponseDto( - modelService.getModels(appGuid, SecurityUtils.getPrincipalId()), - appGuid - ); + return modelService.getModels(appGuid, SecurityUtils.getPrincipalId()); } @ResponseStatus(CREATED) @@ -84,25 +89,19 @@ public List getModels( }) public ModelResponseDto createModel( @ApiParam(value = "GUID of application", required = true, example = GUID_EXAMPLE) - @PathVariable final String appGuid, + @PathVariable + final String appGuid, @ApiParam(value = "Model object that needs to be created", required = true) @Valid - @RequestBody final ModelCreateDto modelCreateDto + @RequestBody + final ModelCreateDto modelCreateDto ) { - Model model; - switch (modelCreateDto.getType()) { - case DETECTION: - model = modelService.createDetectionModel(modelCreateDto, appGuid, SecurityUtils.getPrincipalId()); - break; - case RECOGNITION: - model = modelService.createRecognitionModel(modelCreateDto, appGuid, SecurityUtils.getPrincipalId()); - break; - case VERIFY: - model = modelService.createVerificationModel(modelCreateDto, appGuid, SecurityUtils.getPrincipalId()); - break; - default: - throw new IncorrectModelTypeException(modelCreateDto.getType()); - } + Model model = switch (modelCreateDto.getType()) { + case DETECTION -> modelService.createDetectionModel(modelCreateDto, appGuid, SecurityUtils.getPrincipalId()); + case RECOGNITION -> modelService.createRecognitionModel(modelCreateDto, appGuid, SecurityUtils.getPrincipalId()); + case VERIFY -> modelService.createVerificationModel(modelCreateDto, appGuid, SecurityUtils.getPrincipalId()); + default -> throw new IncorrectModelTypeException(modelCreateDto.getType()); + }; return modelMapper.toResponseDto(model, appGuid); } @@ -114,12 +113,15 @@ public ModelResponseDto createModel( }) public ModelResponseDto cloneModel( @ApiParam(value = "GUID of application", required = true, example = GUID_EXAMPLE) - @PathVariable final String appGuid, + @PathVariable + final String appGuid, @ApiParam(value = "GUID of model that needs to be cloned", required = true, example = GUID_EXAMPLE) - @PathVariable final String guid, + @PathVariable + final String guid, @ApiParam(value = "Model data", required = true) @Valid - @RequestBody final ModelCloneDto modelCloneDto) { + @RequestBody + final ModelCloneDto modelCloneDto) { var clonedModel = modelService.cloneModel(modelCloneDto, appGuid, guid, SecurityUtils.getPrincipalId()); @@ -133,12 +135,15 @@ public ModelResponseDto cloneModel( }) public ModelResponseDto updateModel( @ApiParam(value = "GUID of application", required = true, example = GUID_EXAMPLE) - @PathVariable final String appGuid, + @PathVariable + final String appGuid, @ApiParam(value = "GUID of model that needs to be updated", required = true, example = GUID_EXAMPLE) - @PathVariable final String guid, + @PathVariable + final String guid, @ApiParam(value = "Model data", required = true) @Valid - @RequestBody final ModelUpdateDto modelUpdateDto + @RequestBody + final ModelUpdateDto modelUpdateDto ) { var updatedModel = modelService.updateModel(modelUpdateDto, appGuid, guid, SecurityUtils.getPrincipalId()); @@ -149,23 +154,40 @@ public ModelResponseDto updateModel( @ApiOperation(value = "Generate new api-key for model") public ModelResponseDto regenerateApiKey( @ApiParam(value = "GUID of application", required = true, example = GUID_EXAMPLE) - @PathVariable final String appGuid, + @PathVariable + final String appGuid, @ApiParam(value = "GUID of the model which GUID needs to be regenerated", required = true, example = GUID_EXAMPLE) - @PathVariable final String guid + @PathVariable + final String guid ) { modelService.regenerateApiKey(appGuid, guid, SecurityUtils.getPrincipalId()); - return modelMapper.toResponseDto(modelService.getModel(appGuid, guid, SecurityUtils.getPrincipalId()), appGuid); + return modelService.getModelDto(appGuid, guid, SecurityUtils.getPrincipalId()); } @DeleteMapping("/model/{guid}") @ApiOperation(value = "Delete model") public void deleteModel( @ApiParam(value = "GUID of application", required = true, example = GUID_EXAMPLE) - @PathVariable final String appGuid, + @PathVariable + final String appGuid, @ApiParam(value = "GUID of the model that needs to be deleted", required = true, example = GUID_EXAMPLE) - @PathVariable final String guid + @PathVariable + final String guid ) { modelService.deleteModel(appGuid, guid, SecurityUtils.getPrincipalId()); } -} \ No newline at end of file + + @GetMapping("/model/{guid}/statistics") + @ApiOperation("Get summarized by day statistics of a model for the last couple of months") + public List getSummarizedByDayModelStatistics( + @ApiParam(value = "GUID of application", required = true, example = GUID_EXAMPLE) + @PathVariable + final String appGuid, + @ApiParam(value = "GUID of model", required = true, example = GUID_EXAMPLE) + @PathVariable + final String guid + ) { + return modelService.getSummarizedByDayModelStatistics(appGuid, guid, SecurityUtils.getPrincipalId()); + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/controller/UserController.java b/java/admin/src/main/java/com/exadel/frs/controller/UserController.java index fc9db46438..0fd901ab77 100644 --- a/java/admin/src/main/java/com/exadel/frs/controller/UserController.java +++ b/java/admin/src/main/java/com/exadel/frs/controller/UserController.java @@ -16,30 +16,34 @@ package com.exadel.frs.controller; +import static com.exadel.frs.system.global.Constants.ADMIN; import static com.exadel.frs.system.global.Constants.DEMO_GUID; import static com.exadel.frs.system.global.Constants.GUID_EXAMPLE; import static org.springframework.http.HttpStatus.CREATED; import static org.springframework.http.HttpStatus.FOUND; import static org.springframework.http.HttpStatus.OK; -import com.exadel.frs.dto.ui.ChangePasswordDto; -import com.exadel.frs.dto.ui.UserAutocompleteDto; -import com.exadel.frs.dto.ui.UserCreateDto; -import com.exadel.frs.dto.ui.UserDeleteDto; -import com.exadel.frs.dto.ui.UserResponseDto; -import com.exadel.frs.dto.ui.UserRoleResponseDto; -import com.exadel.frs.dto.ui.UserRoleUpdateDto; -import com.exadel.frs.dto.ui.UserUpdateDto; import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.enums.GlobalRole; import com.exadel.frs.commonservice.enums.Replacer; -import com.exadel.frs.exception.AccessDeniedException; import com.exadel.frs.commonservice.exception.DemoNotAvailableException; +import com.exadel.frs.dto.ChangePasswordDto; +import com.exadel.frs.dto.ForgotPasswordDto; +import com.exadel.frs.dto.ResetPasswordDto; +import com.exadel.frs.dto.UserAutocompleteDto; +import com.exadel.frs.dto.UserCreateDto; +import com.exadel.frs.dto.UserDeleteDto; +import com.exadel.frs.dto.UserResponseDto; +import com.exadel.frs.dto.UserRoleResponseDto; +import com.exadel.frs.dto.UserRoleUpdateDto; +import com.exadel.frs.dto.UserUpdateDto; +import com.exadel.frs.exception.AccessDeniedException; import com.exadel.frs.exception.UserDoesNotExistException; import com.exadel.frs.helpers.SecurityUtils; import com.exadel.frs.mapper.UserGlobalRoleMapper; import com.exadel.frs.mapper.UserMapper; import com.exadel.frs.service.AppService; import com.exadel.frs.service.ModelService; +import com.exadel.frs.service.ResetPasswordTokenService; import com.exadel.frs.service.UserService; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -67,7 +71,7 @@ import org.springframework.web.bind.annotation.RestController; @RestController -@RequestMapping("/user") +@RequestMapping(ADMIN + "/user") @RequiredArgsConstructor public class UserController { @@ -76,6 +80,7 @@ public class UserController { private final AppService appService; private final ModelService modelService; private final UserGlobalRoleMapper userGlobalRoleMapper; + private final ResetPasswordTokenService resetPasswordTokenService; private Environment env; @@ -125,6 +130,7 @@ public void changePassword( public ResponseEntity createUser( @ApiParam(value = "User object that needs to be created", required = true) @RequestBody + @Valid final UserCreateDto userCreateDto ) { User user; @@ -195,7 +201,7 @@ public void confirmRegistration( @RequestParam final String token, final HttpServletResponse response) throws IOException { userService.confirmRegistration(token); - redirectToHomePage(response); + redirectToLoginPage(response); } @GetMapping("/demo/model") @@ -244,9 +250,33 @@ public List getUsers( ); } - private void redirectToHomePage(final HttpServletResponse response) throws IOException { + @PostMapping("/forgot-password") + @ApiOperation("Assigns/Reassigns a reset password token to a user and then sends the token to his email") + public void assignAndSendResetPasswordToken( + @ApiParam(value = "An email of a user", required = true) + @Valid + @RequestBody + final ForgotPasswordDto forgotPasswordDto) { + resetPasswordTokenService.assignAndSendToken(forgotPasswordDto.getEmail()); + } + + @PutMapping("/reset-password") + @ApiOperation("Resets a user's password to a new one") + public void resetPassword( + @ApiParam(value = "A new user password", required = true) + @Valid + @RequestBody + final ResetPasswordDto resetPasswordDto, + @ApiParam(value = "A reset password token", required = true) + @RequestParam + final String token) { + val user = resetPasswordTokenService.exchangeTokenOnUser(token); + userService.resetPassword(user, resetPasswordDto.getPassword()); + } + + private void redirectToLoginPage(final HttpServletResponse response) throws IOException { response.setStatus(FOUND.value()); - val url = env.getProperty("host.frs"); + val url = env.getProperty("host.frs") + "/login"; response.sendRedirect(url); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/AppCreateDto.java b/java/admin/src/main/java/com/exadel/frs/dto/AppCreateDto.java similarity index 68% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/AppCreateDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/AppCreateDto.java index b36abbf970..aafe39d031 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/AppCreateDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/AppCreateDto.java @@ -14,9 +14,12 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; +import static com.exadel.frs.commonservice.system.global.RegExConstants.ALLOWED_SPECIAL_CHARACTERS; import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import javax.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -29,5 +32,7 @@ public class AppCreateDto { @NotBlank(message = "Application name cannot be empty") + @Size(min = 1, max = 50, message = "Application name size must be between 1 and 50") + @Pattern(regexp = ALLOWED_SPECIAL_CHARACTERS, message = "The name cannot contain the following special characters: ';', '/', '\\'") private String name; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/AppOwnerDto.java b/java/admin/src/main/java/com/exadel/frs/dto/AppOwnerDto.java similarity index 95% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/AppOwnerDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/AppOwnerDto.java index 0146864134..aec0324924 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/AppOwnerDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/AppOwnerDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import lombok.Data; @@ -24,4 +24,4 @@ public class AppOwnerDto { private String userId; private String firstName; private String lastName; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/AppResponseDto.java b/java/admin/src/main/java/com/exadel/frs/dto/AppResponseDto.java similarity index 96% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/AppResponseDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/AppResponseDto.java index 963fe0322c..818570e674 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/AppResponseDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/AppResponseDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import lombok.Data; @@ -26,4 +26,4 @@ public class AppResponseDto { private String apiKey; private String role; private AppOwnerDto owner; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/enums/AppModelAccess.java b/java/admin/src/main/java/com/exadel/frs/dto/AppStatusResponseDto.java similarity index 80% rename from java/common/src/main/java/com/exadel/frs/commonservice/enums/AppModelAccess.java rename to java/admin/src/main/java/com/exadel/frs/dto/AppStatusResponseDto.java index 9481e48076..6aa6479ab7 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/enums/AppModelAccess.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/AppStatusResponseDto.java @@ -14,22 +14,19 @@ * permissions and limitations under the License. */ -package com.exadel.frs.commonservice.enums; +package com.exadel.frs.dto; +import com.exadel.frs.commonservice.enums.AppStatus; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +@Getter +@Setter @NoArgsConstructor @AllArgsConstructor -public enum AppModelAccess implements EnumCode { +public class AppStatusResponseDto { - OWNER("O"), - READONLY("R"), - TRAIN("T"); - - @Getter - @Setter - private String code; + private AppStatus status; } diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/AppUpdateDto.java b/java/admin/src/main/java/com/exadel/frs/dto/AppUpdateDto.java similarity index 68% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/AppUpdateDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/AppUpdateDto.java index e8765e34d0..dc82dca6d2 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/AppUpdateDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/AppUpdateDto.java @@ -14,9 +14,12 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; +import static com.exadel.frs.commonservice.system.global.RegExConstants.ALLOWED_SPECIAL_CHARACTERS; import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import javax.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -29,5 +32,7 @@ public class AppUpdateDto { @NotBlank(message = "Application name cannot be empty") + @Size(min = 1, max = 50, message = "Application name size must be between 1 and 50") + @Pattern(regexp = ALLOWED_SPECIAL_CHARACTERS, message = "The name cannot contain the following special characters: ';', '/', '\\'") private String name; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/ChangePasswordDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ChangePasswordDto.java similarity index 86% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/ChangePasswordDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/ChangePasswordDto.java index e118f83f3e..62a0d7338c 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/ChangePasswordDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/ChangePasswordDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.Size; @@ -27,8 +27,8 @@ @AllArgsConstructor public class ChangePasswordDto { - @NotEmpty(message = "User's old password is incorrect") - @Size(min = 8, max = 255, message = "User's old password is incorrect") + @NotEmpty(message = "User's password is incorrect") + @Size(min = 8, max = 255, message = "User's password is incorrect") private String oldPassword; @NotEmpty(message = "User's new password is incorrect") diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ConfigDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ConfigDto.java new file mode 100644 index 0000000000..c8b575bacd --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/dto/ConfigDto.java @@ -0,0 +1,15 @@ +package com.exadel.frs.dto; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ConfigDto { + + private boolean mailServiceEnabled; +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ForgotPasswordDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ForgotPasswordDto.java new file mode 100644 index 0000000000..ff4d53c195 --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/dto/ForgotPasswordDto.java @@ -0,0 +1,19 @@ +package com.exadel.frs.dto; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Size; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class ForgotPasswordDto { + + @NotBlank(message = "User's email cannot be empty") + @Size(min = 1, max = 63, message = "User's email size must be between 1 and 63") + private String email; +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelCloneDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ModelCloneDto.java similarity index 68% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/ModelCloneDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/ModelCloneDto.java index d0abc26e06..54a0c4fa12 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelCloneDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/ModelCloneDto.java @@ -14,15 +14,17 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; +import static com.exadel.frs.commonservice.system.global.RegExConstants.ALLOWED_SPECIAL_CHARACTERS; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import javax.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import javax.validation.constraints.NotBlank; - @Data @Builder @AllArgsConstructor @@ -30,6 +32,7 @@ public class ModelCloneDto { @NotBlank(message = "Model name cannot be empty") + @Size(min = 1, max = 50, message = "Model name size must be between 1 and 50") + @Pattern(regexp = ALLOWED_SPECIAL_CHARACTERS, message = "The name cannot contain the following special characters: ';', '/', '\\'") private String name; - } diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelCreateDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ModelCreateDto.java similarity index 73% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/ModelCreateDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/ModelCreateDto.java index 8e74e261c0..7e05a84621 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelCreateDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/ModelCreateDto.java @@ -14,11 +14,14 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; +import static com.exadel.frs.commonservice.system.global.RegExConstants.ALLOWED_SPECIAL_CHARACTERS; import com.exadel.frs.commonservice.enums.ModelType; import com.exadel.frs.validation.ValidEnum; import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import javax.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -31,8 +34,11 @@ public class ModelCreateDto { @NotBlank(message = "Model name cannot be empty") + @Size(min = 1, max = 50, message = "Model name size must be between 1 and 50") + @Pattern(regexp = ALLOWED_SPECIAL_CHARACTERS, message = "The name cannot contain the following special characters: ';', '/', '\\'") private String name; + @NotBlank(message = "Model Type cannot be empty") @ValidEnum(message = "Model Type '${validatedValue}' doesn't exist!", targetClassType = ModelType.class) private String type; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ModelResponseDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ModelResponseDto.java new file mode 100644 index 0000000000..2338b44cb4 --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/dto/ModelResponseDto.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.exadel.frs.dto; + +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import com.exadel.frs.commonservice.enums.ModelType; +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +import java.time.LocalDateTime; +import lombok.NoArgsConstructor; + +@JsonInclude(NON_NULL) +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ModelResponseDto { + + private String id; + private String name; + private String apiKey; + private ModelType type; + private Long subjectCount; + private Long imageCount; + private LocalDateTime createdDate; +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelUpdateDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ModelUpdateDto.java similarity index 68% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/ModelUpdateDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/ModelUpdateDto.java index b92320a2a2..2fb0725623 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelUpdateDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/ModelUpdateDto.java @@ -14,9 +14,12 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; +import static com.exadel.frs.commonservice.system.global.RegExConstants.ALLOWED_SPECIAL_CHARACTERS; import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import javax.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -29,5 +32,7 @@ public class ModelUpdateDto { @NotBlank(message = "Model name cannot be empty") + @Size(min = 1, max = 50, message = "Model name size must be between 1 and 50") + @Pattern(regexp = ALLOWED_SPECIAL_CHARACTERS, message = "The name cannot contain the following special characters: ';', '/', '\\'") private String name; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ResetPasswordDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ResetPasswordDto.java new file mode 100644 index 0000000000..340ec95f4a --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/dto/ResetPasswordDto.java @@ -0,0 +1,19 @@ +package com.exadel.frs.dto; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Size; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class ResetPasswordDto { + + @NotBlank(message = "The password cannot be blank") + @Size(min = 8, max = 255, message = "The password size must be between 8 and 255 characters") + private String password; +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserAutocompleteDto.java b/java/admin/src/main/java/com/exadel/frs/dto/UserAutocompleteDto.java similarity index 96% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/UserAutocompleteDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/UserAutocompleteDto.java index f4d62b1306..b9ff27da97 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserAutocompleteDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/UserAutocompleteDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import java.util.List; import lombok.AllArgsConstructor; @@ -31,4 +31,4 @@ public class UserAutocompleteDto { private String query; private int length; private List results; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserCreateDto.java b/java/admin/src/main/java/com/exadel/frs/dto/UserCreateDto.java similarity index 71% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/UserCreateDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/UserCreateDto.java index 5973d284f9..961721491d 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserCreateDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/UserCreateDto.java @@ -14,9 +14,11 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import com.fasterxml.jackson.annotation.JsonProperty; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -29,9 +31,17 @@ public class UserCreateDto { private String email; + + @NotBlank(message = "User's first name cannot be empty") + @Size(min = 1, max = 50, message = "User's first name size must be between 1 and 50") private String firstName; + + @NotBlank(message = "User's last name cannot be empty") + @Size(min = 1, max = 50, message = "User's last name size must be between 1 and 50") private String lastName; + private String password; + @JsonProperty(defaultValue = "true") private boolean isAllowStatistics; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserDeleteDto.java b/java/admin/src/main/java/com/exadel/frs/dto/UserDeleteDto.java similarity index 97% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/UserDeleteDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/UserDeleteDto.java index ff30607376..0c3842ec0e 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserDeleteDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/UserDeleteDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.enums.Replacer; diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserInviteDto.java b/java/admin/src/main/java/com/exadel/frs/dto/UserInviteDto.java similarity index 96% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/UserInviteDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/UserInviteDto.java index 729620b5d9..41bf26169d 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserInviteDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/UserInviteDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import lombok.AllArgsConstructor; import lombok.Builder; @@ -29,4 +29,4 @@ public class UserInviteDto { private String role; private String userEmail; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserResponseDto.java b/java/admin/src/main/java/com/exadel/frs/dto/UserResponseDto.java similarity index 96% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/UserResponseDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/UserResponseDto.java index ebd27672a7..e2c97cdffb 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserResponseDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/UserResponseDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import lombok.Data; @@ -25,4 +25,4 @@ public class UserResponseDto { private String email; private String firstName; private String lastName; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserRoleResponseDto.java b/java/admin/src/main/java/com/exadel/frs/dto/UserRoleResponseDto.java similarity index 96% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/UserRoleResponseDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/UserRoleResponseDto.java index 2740b63f9d..0fc75c571b 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserRoleResponseDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/UserRoleResponseDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import lombok.Data; @@ -26,4 +26,4 @@ public class UserRoleResponseDto { private String lastName; private String role; private String email; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserRoleUpdateDto.java b/java/admin/src/main/java/com/exadel/frs/dto/UserRoleUpdateDto.java similarity index 97% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/UserRoleUpdateDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/UserRoleUpdateDto.java index 3ea4813f81..02b0af219d 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserRoleUpdateDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/UserRoleUpdateDto.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; import com.exadel.frs.commonservice.enums.GlobalRole; import com.exadel.frs.validation.ValidEnum; @@ -36,4 +36,4 @@ public class UserRoleUpdateDto { @NotBlank(message = "Role cannot be empty") @ValidEnum(message = "Global role '${validatedValue}' doesn't exist!", targetClassType = GlobalRole.class) private String role; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserUpdateDto.java b/java/admin/src/main/java/com/exadel/frs/dto/UserUpdateDto.java similarity index 67% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/UserUpdateDto.java rename to java/admin/src/main/java/com/exadel/frs/dto/UserUpdateDto.java index a0d62ace5e..46ea583207 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserUpdateDto.java +++ b/java/admin/src/main/java/com/exadel/frs/dto/UserUpdateDto.java @@ -14,9 +14,10 @@ * permissions and limitations under the License. */ -package com.exadel.frs.dto.ui; +package com.exadel.frs.dto; -import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -28,9 +29,11 @@ @NoArgsConstructor public class UserUpdateDto { - @NotEmpty(message = "User's first name is incorrect") + @NotBlank(message = "Field firstName cannot be empty") + @Size(min = 1, max = 50, message = "User's first name size must be between 1 and 50") private String firstName; - @NotEmpty(message = "User's last name is incorrect") + @NotBlank(message = "Field lastName cannot be empty") + @Size(min = 1, max = 50, message = "User's last name size must be between 1 and 50") private String lastName; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserUpdateResponseDto.java b/java/admin/src/main/java/com/exadel/frs/dto/ui/UserUpdateResponseDto.java deleted file mode 100644 index 51ab17fda0..0000000000 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/UserUpdateResponseDto.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.dto.ui; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -public class UserUpdateResponseDto { - - private String firstName; - private String lastName; -} \ No newline at end of file diff --git a/java/admin/src/main/java/com/exadel/frs/exception/AccessDeniedException.java b/java/admin/src/main/java/com/exadel/frs/exception/AccessDeniedException.java index 2d913464bb..ccb20b8b2c 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/AccessDeniedException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/AccessDeniedException.java @@ -27,4 +27,4 @@ public class AccessDeniedException extends BasicException { public AccessDeniedException() { super(APP_ACCESS_DENIED, MESSAGE); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/AppNotFoundException.java b/java/admin/src/main/java/com/exadel/frs/exception/AppNotFoundException.java index 9033fec961..91b4f34f25 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/AppNotFoundException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/AppNotFoundException.java @@ -28,4 +28,4 @@ public class AppNotFoundException extends BasicException { public AppNotFoundException(final String guid) { super(APP_NOT_FOUND, format(MESSAGE, guid)); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/EmailAlreadyRegisteredException.java b/java/admin/src/main/java/com/exadel/frs/exception/EmailAlreadyRegisteredException.java index 3c306c8db9..bc2383b910 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/EmailAlreadyRegisteredException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/EmailAlreadyRegisteredException.java @@ -27,4 +27,4 @@ public class EmailAlreadyRegisteredException extends BasicException { public EmailAlreadyRegisteredException() { super(EMAIL_ALREADY_REGISTERED, MESSAGE); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/IncorrectAccessTypeException.java b/java/admin/src/main/java/com/exadel/frs/exception/IncorrectAccessTypeException.java deleted file mode 100644 index a56495f532..0000000000 --- a/java/admin/src/main/java/com/exadel/frs/exception/IncorrectAccessTypeException.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.exception; - -import com.exadel.frs.commonservice.exception.BasicException; - -import static com.exadel.frs.commonservice.handler.CommonExceptionCode.INCORRECT_ACCESS_TYPE; -import static java.lang.String.format; - -public class IncorrectAccessTypeException extends BasicException { - - public static final String ACCESS_TYPE_NOT_EXISTS_MESSAGE = "Access type %s does not exists"; - - public IncorrectAccessTypeException(final String accessType) { - super(INCORRECT_ACCESS_TYPE, format(ACCESS_TYPE_NOT_EXISTS_MESSAGE, accessType)); - } -} \ No newline at end of file diff --git a/java/admin/src/main/java/com/exadel/frs/exception/IncorrectStatisticsTypeException.java b/java/admin/src/main/java/com/exadel/frs/exception/IncorrectStatisticsTypeException.java deleted file mode 100644 index cb374825bd..0000000000 --- a/java/admin/src/main/java/com/exadel/frs/exception/IncorrectStatisticsTypeException.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.exadel.frs.exception; - -import com.exadel.frs.commonservice.exception.BasicException; - -import static com.exadel.frs.commonservice.handler.CrudExceptionCode.INCORRECT_STATISTICS_ROLE; -import static java.lang.String.format; - -public class IncorrectStatisticsTypeException extends BasicException { - - public static final String STATISTICS_TYPE_NOT_EXISTS_MESSAGE = "Statistics type %s does not exists"; - - public IncorrectStatisticsTypeException(final String statisticsType) { - super(INCORRECT_STATISTICS_ROLE, format(STATISTICS_TYPE_NOT_EXISTS_MESSAGE, statisticsType)); - } -} \ No newline at end of file diff --git a/java/admin/src/main/java/com/exadel/frs/exception/IncorrectUserPasswordException.java b/java/admin/src/main/java/com/exadel/frs/exception/IncorrectUserPasswordException.java index 1034795da0..98d89ba6fc 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/IncorrectUserPasswordException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/IncorrectUserPasswordException.java @@ -25,4 +25,4 @@ public class IncorrectUserPasswordException extends BasicException { public IncorrectUserPasswordException() { super(INCORRECT_USER_PASSWORD, "User's password is incorrect"); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/InsufficientPrivilegesException.java b/java/admin/src/main/java/com/exadel/frs/exception/InsufficientPrivilegesException.java index 38721c3e6f..bf8005a4f9 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/InsufficientPrivilegesException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/InsufficientPrivilegesException.java @@ -31,4 +31,4 @@ public InsufficientPrivilegesException() { public InsufficientPrivilegesException(final String message) { super(INSUFFICIENT_PRIVILEGES, message); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/InvalidEmailException.java b/java/admin/src/main/java/com/exadel/frs/exception/InvalidEmailException.java index 83aca66399..75ce28ac94 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/InvalidEmailException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/InvalidEmailException.java @@ -27,4 +27,4 @@ public class InvalidEmailException extends BasicException { public InvalidEmailException() { super(INVALID_EMAIL_FORMAT, MESSAGE); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/InvalidResetPasswordTokenException.java b/java/admin/src/main/java/com/exadel/frs/exception/InvalidResetPasswordTokenException.java new file mode 100644 index 0000000000..d2270a46e1 --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/exception/InvalidResetPasswordTokenException.java @@ -0,0 +1,13 @@ +package com.exadel.frs.exception; + +import static com.exadel.frs.commonservice.handler.CrudExceptionCode.INVALID_RESET_PASSWORD_TOKEN; +import com.exadel.frs.commonservice.exception.BasicException; + +public class InvalidResetPasswordTokenException extends BasicException { + + public static final String MESSAGE = "The reset password token is invalid!"; + + public InvalidResetPasswordTokenException() { + super(INVALID_RESET_PASSWORD_TOKEN, MESSAGE); + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/MailServerDisabledException.java b/java/admin/src/main/java/com/exadel/frs/exception/MailServerDisabledException.java new file mode 100644 index 0000000000..acf2cdfb7c --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/exception/MailServerDisabledException.java @@ -0,0 +1,13 @@ +package com.exadel.frs.exception; + +import static com.exadel.frs.commonservice.handler.CrudExceptionCode.MAIL_SERVER_EXCEPTION; +import com.exadel.frs.commonservice.exception.BasicException; + +public class MailServerDisabledException extends BasicException { + + public static final String MESSAGE = "We cannot send an email. No email server enabled!"; + + public MailServerDisabledException() { + super(MAIL_SERVER_EXCEPTION, MESSAGE); + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/ModelDoesNotBelongToAppException.java b/java/admin/src/main/java/com/exadel/frs/exception/ModelDoesNotBelongToAppException.java index 012b770c39..06d0d2fc04 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/ModelDoesNotBelongToAppException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/ModelDoesNotBelongToAppException.java @@ -28,4 +28,4 @@ public class ModelDoesNotBelongToAppException extends BasicException { public ModelDoesNotBelongToAppException(String modelGuid, String appGuid) { super(MODEL_DOES_NOT_BELONG_TO_APP, format(MESSAGE, modelGuid, appGuid)); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/ModelShareRequestNotFoundException.java b/java/admin/src/main/java/com/exadel/frs/exception/ModelShareRequestNotFoundException.java deleted file mode 100644 index 08c3789983..0000000000 --- a/java/admin/src/main/java/com/exadel/frs/exception/ModelShareRequestNotFoundException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.exception; - -import com.exadel.frs.commonservice.exception.BasicException; - -import java.util.UUID; - -import static com.exadel.frs.commonservice.handler.CrudExceptionCode.MODEL_SHARE_REQUEST_NOT_FOUND; -import static java.lang.String.format; - -public class ModelShareRequestNotFoundException extends BasicException { - - public static final String MESSAGE = "Model share request %s not found. It might be expired."; - - public ModelShareRequestNotFoundException(final UUID requestId) { - super(MODEL_SHARE_REQUEST_NOT_FOUND, format(MESSAGE, requestId.toString())); - } -} \ No newline at end of file diff --git a/java/admin/src/main/java/com/exadel/frs/exception/NameIsNotUniqueException.java b/java/admin/src/main/java/com/exadel/frs/exception/NameIsNotUniqueException.java index 7a621cd474..5bc9d81fa3 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/NameIsNotUniqueException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/NameIsNotUniqueException.java @@ -28,4 +28,4 @@ public class NameIsNotUniqueException extends BasicException { public NameIsNotUniqueException(String fieldName) { super(NAME_IS_NOT_UNIQUE, format(MESSAGE, fieldName)); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/RegistrationTokenExpiredException.java b/java/admin/src/main/java/com/exadel/frs/exception/RegistrationTokenExpiredException.java index 741b350f13..5c0caae951 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/RegistrationTokenExpiredException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/RegistrationTokenExpiredException.java @@ -27,4 +27,4 @@ public class RegistrationTokenExpiredException extends BasicException { public RegistrationTokenExpiredException() { super(USER_REGISTRATION_TOKEN_EXPIRED, MESSAGE); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/SelfRoleChangeException.java b/java/admin/src/main/java/com/exadel/frs/exception/SelfRoleChangeException.java index 4999d3a59f..aecae5290f 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/SelfRoleChangeException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/SelfRoleChangeException.java @@ -27,4 +27,4 @@ public class SelfRoleChangeException extends BasicException { public SelfRoleChangeException() { super(SELF_ROLE_CHANGE, MESSAGE); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/UnreachableEmailException.java b/java/admin/src/main/java/com/exadel/frs/exception/UnreachableEmailException.java index 9feb9589ab..bbcec333d3 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/UnreachableEmailException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/UnreachableEmailException.java @@ -27,4 +27,4 @@ public class UnreachableEmailException extends BasicException { public UnreachableEmailException(String email) { super(UNREACHABLE_EMAIL_ADDRESS, String.format(MESSAGE, email)); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/UserAlreadyHasAccessToAppException.java b/java/admin/src/main/java/com/exadel/frs/exception/UserAlreadyHasAccessToAppException.java index 27ac754198..e57403a4a2 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/UserAlreadyHasAccessToAppException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/UserAlreadyHasAccessToAppException.java @@ -25,7 +25,7 @@ public class UserAlreadyHasAccessToAppException extends BasicException { private static final String MESSAGE = "User %s already has access to application %s"; - public UserAlreadyHasAccessToAppException(String user, String appGuid) { - super(USER_ALREADY_HAS_ACCESS_TO_APP, format(MESSAGE, user, appGuid)); + public UserAlreadyHasAccessToAppException(String user, String appName) { + super(USER_ALREADY_HAS_ACCESS_TO_APP, format(MESSAGE, user, appName)); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/exception/UserDoesNotExistException.java b/java/admin/src/main/java/com/exadel/frs/exception/UserDoesNotExistException.java index 7fb080bbad..83dc6a57a1 100644 --- a/java/admin/src/main/java/com/exadel/frs/exception/UserDoesNotExistException.java +++ b/java/admin/src/main/java/com/exadel/frs/exception/UserDoesNotExistException.java @@ -28,4 +28,4 @@ public class UserDoesNotExistException extends BasicException { public UserDoesNotExistException(final String userId) { super(USER_DOES_NOT_EXIST, format(MESSAGE, userId)); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/helpers/EmailSender.java b/java/admin/src/main/java/com/exadel/frs/helpers/EmailSender.java index 1714ee7ccc..590365ade2 100644 --- a/java/admin/src/main/java/com/exadel/frs/helpers/EmailSender.java +++ b/java/admin/src/main/java/com/exadel/frs/helpers/EmailSender.java @@ -45,7 +45,7 @@ public void sendMail(final String to, final String subject, final String message } helper.setTo(to); helper.setSubject(subject); - helper.setText(message); + helper.setText(message, true); try { javaMailSender.send(msg); @@ -53,4 +53,4 @@ public void sendMail(final String to, final String subject, final String message throw new UnreachableEmailException(to); } } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/helpers/SecurityUtils.java b/java/admin/src/main/java/com/exadel/frs/helpers/SecurityUtils.java index a36a971497..81e6fd97cf 100644 --- a/java/admin/src/main/java/com/exadel/frs/helpers/SecurityUtils.java +++ b/java/admin/src/main/java/com/exadel/frs/helpers/SecurityUtils.java @@ -30,8 +30,4 @@ public User getPrincipal() { public Long getPrincipalId() { return getPrincipal().getId(); } - - public boolean isAnonymousUser() { - return !SecurityContextHolder.getContext().getAuthentication().isAuthenticated(); - } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/mapper/AppMapper.java b/java/admin/src/main/java/com/exadel/frs/mapper/AppMapper.java index 014eeec542..e90c69d213 100644 --- a/java/admin/src/main/java/com/exadel/frs/mapper/AppMapper.java +++ b/java/admin/src/main/java/com/exadel/frs/mapper/AppMapper.java @@ -16,7 +16,7 @@ package com.exadel.frs.mapper; -import com.exadel.frs.dto.ui.AppResponseDto; +import com.exadel.frs.dto.AppResponseDto; import com.exadel.frs.commonservice.entity.App; import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.entity.UserAppRole; @@ -50,4 +50,4 @@ default String getRole(App app, @Context Long userId) { .map(Enum::name) .orElse(null); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/mapper/MlModelMapper.java b/java/admin/src/main/java/com/exadel/frs/mapper/MlModelMapper.java index 1e97d4e1ac..d24caf19f5 100644 --- a/java/admin/src/main/java/com/exadel/frs/mapper/MlModelMapper.java +++ b/java/admin/src/main/java/com/exadel/frs/mapper/MlModelMapper.java @@ -16,15 +16,13 @@ package com.exadel.frs.mapper; -import com.exadel.frs.dto.ui.ModelResponseDto; -import com.exadel.frs.commonservice.entity.AppModel; import com.exadel.frs.commonservice.entity.Model; -import com.exadel.frs.commonservice.enums.AppModelAccess; +import com.exadel.frs.commonservice.projection.ModelProjection; +import com.exadel.frs.dto.ModelResponseDto; import java.util.List; import org.mapstruct.Context; import org.mapstruct.Mapper; import org.mapstruct.Mapping; -import org.mapstruct.Named; @Mapper public interface MlModelMapper { @@ -32,5 +30,8 @@ public interface MlModelMapper { @Mapping(source = "guid", target = "id") ModelResponseDto toResponseDto(Model model, @Context String appGuid); + @Mapping(source = "guid", target = "id") + ModelResponseDto toResponseDto(ModelProjection model, @Context String appGuid); + List toResponseDto(List model, @Context String appGuid); -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/mapper/UserAppRoleMapper.java b/java/admin/src/main/java/com/exadel/frs/mapper/UserAppRoleMapper.java index ad83231d20..b6c19a9738 100644 --- a/java/admin/src/main/java/com/exadel/frs/mapper/UserAppRoleMapper.java +++ b/java/admin/src/main/java/com/exadel/frs/mapper/UserAppRoleMapper.java @@ -16,7 +16,7 @@ package com.exadel.frs.mapper; -import com.exadel.frs.dto.ui.UserRoleResponseDto; +import com.exadel.frs.dto.UserRoleResponseDto; import com.exadel.frs.commonservice.entity.UserAppRole; import java.util.List; import org.mapstruct.Mapper; @@ -32,4 +32,4 @@ public interface UserAppRoleMapper { UserRoleResponseDto toUserRoleResponseDto(UserAppRole userAppRole); List toUserRoleResponseDto(List userAppRoles); -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/mapper/UserGlobalRoleMapper.java b/java/admin/src/main/java/com/exadel/frs/mapper/UserGlobalRoleMapper.java index 4016adca15..f6536af030 100644 --- a/java/admin/src/main/java/com/exadel/frs/mapper/UserGlobalRoleMapper.java +++ b/java/admin/src/main/java/com/exadel/frs/mapper/UserGlobalRoleMapper.java @@ -16,7 +16,7 @@ package com.exadel.frs.mapper; -import com.exadel.frs.dto.ui.UserRoleResponseDto; +import com.exadel.frs.dto.UserRoleResponseDto; import com.exadel.frs.commonservice.entity.User; import java.util.List; import org.mapstruct.Mapper; @@ -33,4 +33,4 @@ public interface UserGlobalRoleMapper { UserRoleResponseDto toUserRoleResponseDto(User userAppRole); List toUserRoleResponseDto(List userAppRoles); -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/mapper/UserMapper.java b/java/admin/src/main/java/com/exadel/frs/mapper/UserMapper.java index 687dbff22f..36c9b82b97 100644 --- a/java/admin/src/main/java/com/exadel/frs/mapper/UserMapper.java +++ b/java/admin/src/main/java/com/exadel/frs/mapper/UserMapper.java @@ -16,8 +16,8 @@ package com.exadel.frs.mapper; -import com.exadel.frs.dto.ui.AppOwnerDto; -import com.exadel.frs.dto.ui.UserResponseDto; +import com.exadel.frs.dto.AppOwnerDto; +import com.exadel.frs.dto.UserResponseDto; import com.exadel.frs.commonservice.entity.User; import java.util.List; import org.mapstruct.Mapper; diff --git a/java/admin/src/main/java/com/exadel/frs/repository/AppRepository.java b/java/admin/src/main/java/com/exadel/frs/repository/AppRepository.java index 2619ec8ad2..b0688292e3 100644 --- a/java/admin/src/main/java/com/exadel/frs/repository/AppRepository.java +++ b/java/admin/src/main/java/com/exadel/frs/repository/AppRepository.java @@ -28,4 +28,6 @@ public interface AppRepository extends JpaRepository { List findAllByUserAppRoles_Id_UserId(Long userId); boolean existsByName(String name); -} \ No newline at end of file + + List findAllByOrderByNameAsc(); +} diff --git a/java/admin/src/main/java/com/exadel/frs/repository/ModelShareRequestRepository.java b/java/admin/src/main/java/com/exadel/frs/repository/ModelShareRequestRepository.java deleted file mode 100644 index 19c7614e78..0000000000 --- a/java/admin/src/main/java/com/exadel/frs/repository/ModelShareRequestRepository.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.repository; - -import com.exadel.frs.commonservice.entity.ModelShareRequest; -import com.exadel.frs.commonservice.entity.ModelShareRequestId; -import java.util.UUID; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Query; - -public interface ModelShareRequestRepository extends JpaRepository { - - @Query("select m from ModelShareRequest m where m.id.requestId = :requestId") - ModelShareRequest findModelShareRequestByRequestId(UUID requestId); -} \ No newline at end of file diff --git a/java/admin/src/main/java/com/exadel/frs/repository/ResetPasswordTokenRepository.java b/java/admin/src/main/java/com/exadel/frs/repository/ResetPasswordTokenRepository.java new file mode 100644 index 0000000000..25786afe84 --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/repository/ResetPasswordTokenRepository.java @@ -0,0 +1,15 @@ +package com.exadel.frs.repository; + +import com.exadel.frs.commonservice.entity.ResetPasswordToken; +import java.time.LocalDateTime; +import java.util.UUID; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface ResetPasswordTokenRepository extends JpaRepository { + + void deleteByUserEmail(String email); + + void deleteAllByExpiresInBefore(LocalDateTime dateTime); +} diff --git a/java/admin/src/main/java/com/exadel/frs/service/AppService.java b/java/admin/src/main/java/com/exadel/frs/service/AppService.java index 26dc698a48..6206e960a9 100644 --- a/java/admin/src/main/java/com/exadel/frs/service/AppService.java +++ b/java/admin/src/main/java/com/exadel/frs/service/AppService.java @@ -23,10 +23,10 @@ import static org.apache.commons.lang3.BooleanUtils.isNotTrue; import static org.apache.commons.lang3.StringUtils.isNotEmpty; import com.exadel.frs.commonservice.annotation.CollectStatistics; -import com.exadel.frs.dto.ui.AppCreateDto; -import com.exadel.frs.dto.ui.AppUpdateDto; -import com.exadel.frs.dto.ui.UserInviteDto; -import com.exadel.frs.dto.ui.UserRoleUpdateDto; +import com.exadel.frs.dto.AppCreateDto; +import com.exadel.frs.dto.AppUpdateDto; +import com.exadel.frs.dto.UserInviteDto; +import com.exadel.frs.dto.UserRoleUpdateDto; import com.exadel.frs.commonservice.entity.App; import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.entity.UserAppRole; @@ -34,7 +34,6 @@ import com.exadel.frs.exception.AppNotFoundException; import com.exadel.frs.exception.InsufficientPrivilegesException; import com.exadel.frs.exception.NameIsNotUniqueException; -import com.exadel.frs.exception.SelfRoleChangeException; import com.exadel.frs.exception.UserAlreadyHasAccessToAppException; import com.exadel.frs.repository.AppRepository; import com.exadel.frs.system.security.AuthorizationManager; @@ -101,7 +100,7 @@ public List getApps(final Long userId) { return appRepository.findAllByUserAppRoles_Id_UserId(userId); } - return appRepository.findAll(); + return appRepository.findAllByOrderByNameAsc(); } public AppRole[] getAppRolesToAssign(final String appGuid, final Long userId) { @@ -154,7 +153,7 @@ public UserAppRole inviteUser( val userAppRole = app.getUserAppRole(user.getId()); if (userAppRole.isPresent()) { - throw new UserAlreadyHasAccessToAppException(userInviteDto.getUserEmail(), appGuid); + throw new UserAlreadyHasAccessToAppException(userInviteDto.getUserEmail(), app.getName()); } val appRole = AppRole.valueOf(userInviteDto.getRole()); @@ -209,14 +208,9 @@ public UserAppRole updateUserAppRole(final UserRoleUpdateDto userRoleUpdateDto, authManager.verifyWritePrivilegesToApp(admin, app, true); val userToUpdate = userService.getUserByGuid(userRoleUpdateDto.getUserId()); - if (userToUpdate.getId().equals(adminId)) { - throw new SelfRoleChangeException(); - } - val userToUpdateAppRole = app.getUserAppRole(userToUpdate.getId()).orElseThrow(); val newAppRole = AppRole.valueOf(userRoleUpdateDto.getRole()); - if (userToUpdateAppRole.getRole().equals(OWNER)) { throw new InsufficientPrivilegesException(); } @@ -264,4 +258,4 @@ public void deleteApp(final String guid, final Long userId) { appRepository.deleteById(app.getId()); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/service/ModelCloneService.java b/java/admin/src/main/java/com/exadel/frs/service/ModelCloneService.java index 2ce3041fab..b3e3406d8f 100644 --- a/java/admin/src/main/java/com/exadel/frs/service/ModelCloneService.java +++ b/java/admin/src/main/java/com/exadel/frs/service/ModelCloneService.java @@ -18,7 +18,7 @@ import com.exadel.frs.commonservice.entity.Model; import com.exadel.frs.commonservice.repository.ModelRepository; -import com.exadel.frs.dto.ui.ModelCloneDto; +import com.exadel.frs.dto.ModelCloneDto; import javax.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -37,9 +37,7 @@ public Model cloneModel(Model model, ModelCloneDto modelCloneDto) { val clone = new Model(model); clone.setId(null); clone.setName(modelCloneDto.getName()); - clone.setAppModelAccess(null); return modelRepository.save(clone); } - -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/service/ModelService.java b/java/admin/src/main/java/com/exadel/frs/service/ModelService.java index bb67ba8364..248cb75ad8 100644 --- a/java/admin/src/main/java/com/exadel/frs/service/ModelService.java +++ b/java/admin/src/main/java/com/exadel/frs/service/ModelService.java @@ -16,34 +16,52 @@ package com.exadel.frs.service; +import static java.time.LocalDateTime.now; +import static java.time.ZoneOffset.UTC; +import static java.util.UUID.randomUUID; import com.exadel.frs.commonservice.annotation.CollectStatistics; -import com.exadel.frs.commonservice.entity.*; +import com.exadel.frs.commonservice.entity.App; +import com.exadel.frs.commonservice.entity.Model; +import com.exadel.frs.commonservice.projection.ModelStatisticProjection; +import com.exadel.frs.commonservice.entity.Subject; +import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.enums.ModelType; import com.exadel.frs.commonservice.enums.StatisticsType; import com.exadel.frs.commonservice.exception.ModelNotFoundException; +import com.exadel.frs.commonservice.repository.ImgRepository; import com.exadel.frs.commonservice.repository.ModelRepository; +import com.exadel.frs.commonservice.repository.ModelStatisticRepository; import com.exadel.frs.commonservice.repository.SubjectRepository; -import com.exadel.frs.dto.ui.ModelCloneDto; -import com.exadel.frs.dto.ui.ModelCreateDto; -import com.exadel.frs.dto.ui.ModelUpdateDto; +import com.exadel.frs.dto.ModelCloneDto; +import com.exadel.frs.dto.ModelCreateDto; +import com.exadel.frs.dto.ModelResponseDto; +import com.exadel.frs.dto.ModelUpdateDto; import com.exadel.frs.exception.NameIsNotUniqueException; +import com.exadel.frs.mapper.MlModelMapper; import com.exadel.frs.system.security.AuthorizationManager; +import java.time.LocalDate; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import javax.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; -import javax.transaction.Transactional; -import java.util.*; - -import static java.util.UUID.randomUUID; - @Service @RequiredArgsConstructor @Slf4j public class ModelService { + @Value("${statistic.model.months}") + private int statisticMonths; + private final ModelRepository modelRepository; private final AppService appService; private final AuthorizationManager authManager; @@ -51,19 +69,36 @@ public class ModelService { private final SubjectRepository subjectRepository; private final JdbcTemplate jdbcTemplate; private final ModelCloneService modelCloneService; + private final MlModelMapper modelMapper; + private final ImgRepository imgRepository; + private final ModelStatisticRepository statisticRepository; public Model getModel(final String modelGuid) { return modelRepository.findByGuid(modelGuid) .orElseThrow(() -> new ModelNotFoundException(modelGuid, "")); } + private void validateName(final String newName, final String oldName, final Long appId) { + if (oldName.equals(newName)) { + throw new NameIsNotUniqueException(newName); + } + boolean hasNewNameEntryInDb = modelRepository.existsByUniqueNameAndAppId(newName, appId); + if (hasNewNameEntryInDb) { + boolean hasNewNameMoreThanOneEntryInDb = modelRepository.countByUniqueNameAndAppId(newName, appId) > 1; + boolean isNotEqualsIgnoreCase = !oldName.equalsIgnoreCase(newName); + if (hasNewNameMoreThanOneEntryInDb || isNotEqualsIgnoreCase) { + throw new NameIsNotUniqueException(newName); + } + } + } + private void verifyNameIsUnique(final String name, final Long appId) { - if (modelRepository.existsByNameAndAppId(name, appId)) { + if (modelRepository.existsByUniqueNameAndAppId(name, appId)) { throw new NameIsNotUniqueException(name); } } - public Model getModel(final String appGuid, final String modelGuid, final Long userId) { + private Model getModel(final String appGuid, final String modelGuid, final Long userId) { val model = getModel(modelGuid); val user = userService.getUser(userId); @@ -73,13 +108,31 @@ public Model getModel(final String appGuid, final String modelGuid, final Long u return model; } - public List getModels(final String appGuid, final Long userId) { + public ModelResponseDto getModelDto(final String appGuid, final String modelGuid, final Long userId) { + Model model = getModel(appGuid, modelGuid, userId); + return getModelResponseDto(appGuid, model); + } + + private ModelResponseDto getModelResponseDto(String appGuid, Model model) { + String apiKey = model.getApiKey(); + Long subjectCount = subjectRepository.countAllByApiKey(apiKey); + Long imageCount = imgRepository.getImageCountByApiKey(apiKey); + + ModelResponseDto modelResponseDto = modelMapper.toResponseDto(model, appGuid); + modelResponseDto.setSubjectCount(subjectCount); + modelResponseDto.setImageCount(imageCount); + return modelResponseDto; + } + + public List getModels(final String appGuid, final Long userId) { val app = appService.getApp(appGuid); val user = userService.getUser(userId); authManager.verifyReadPrivilegesToApp(user, app); - return modelRepository.findAllByAppId(app.getId()); + return modelRepository.findAllByAppId(app.getId()) + .stream() + .map(model -> modelMapper.toResponseDto(model, model.apiKey())).collect(Collectors.toList()); } private Model createModel(final ModelCreateDto modelCreateDto, final String appGuid, final Long userId) { @@ -102,6 +155,7 @@ public Model buildModel(ModelCreateDto modelCreateDto, App app) { .apiKey(randomUUID().toString()) .app(app) .type(ModelType.valueOf(modelCreateDto.getType())) + .createdDate(now()) .build(); } @@ -142,27 +196,12 @@ public Model cloneModel( val clonedModel = modelCloneService.cloneModel(model, modelCloneDto); - List clonedAppModelAccessList = cloneAppModels(model, clonedModel); - clonedModel.setAppModelAccess(clonedAppModelAccessList); - // caution: time consuming operation cloneSubjects(model.getApiKey(), clonedModel.getApiKey()); return clonedModel; } - private List cloneAppModels(final Model model, final Model clonedModel) { - val cloneAppModelAccessList = new ArrayList(); - for (val appModel : model.getAppModelAccess()) { - AppModel cloneAppModelAccess = new AppModel(appModel); - cloneAppModelAccess.setId(new AppModelId(clonedModel.getApp().getId(), clonedModel.getId())); - cloneAppModelAccess.setModel(clonedModel); - - cloneAppModelAccessList.add(cloneAppModelAccess); - } - return cloneAppModelAccessList; - } - @Transactional public void cloneSubjects(final String sourceApiKey, final String newApiKey) { subjectRepository @@ -192,14 +231,19 @@ private void cloneSubject(Subject subject, String newApiKey) { } ); - String sql = "select " + - " e.id as embedding_id, " + - " i.id as img_id " + - " from " + - " embedding e left join img i on e.img_id = i.id " + - " inner join subject s on s.id = e.subject_id " + - " where " + - " s.id = ?"; + String sql = """ + select + e.id as embedding_id, + i.id as img_id + from + embedding e + left join + img i on e.img_id = i.id + inner join + subject s on s.id = e.subject_id + where + s.id = ? + """; jdbcTemplate.query( sql, @@ -209,7 +253,10 @@ private void cloneSubject(Subject subject, String newApiKey) { var sourceImgId = rc.getObject("img_id", UUID.class); // could be null (for demo embeddings) jdbcTemplate.update( "insert into embedding(id, subject_id, embedding, calculator, img_id) select ?, ?, e.embedding, e.calculator, ? from embedding e where e.id = ?", - UUID.randomUUID(), newSubjectId, sourceImgId2NewImgId.get(sourceImgId), sourceEmbeddingId + UUID.randomUUID(), + newSubjectId, + sourceImgId2NewImgId.get(sourceImgId), + sourceEmbeddingId ); } ); @@ -226,10 +273,8 @@ public Model updateModel( authManager.verifyWritePrivilegesToApp(user, model.getApp()); - if (!model.getName().equals(modelUpdateDto.getName())) { - verifyNameIsUnique(modelUpdateDto.getName(), model.getApp().getId()); - model.setName(modelUpdateDto.getName()); - } + validateName(modelUpdateDto.getName(), model.getName(), model.getApp().getId()); + model.setName(modelUpdateDto.getName()); return modelRepository.save(model); } @@ -256,4 +301,18 @@ public void deleteModel(final String appGuid, final String guid, final Long user modelRepository.deleteById(model.getId()); } -} \ No newline at end of file + + public List getSummarizedByDayModelStatistics(final String appGuid, final String guid, final Long userId) { + val model = getModel(guid); + val user = userService.getUser(userId); + + authManager.verifyReadPrivilegesToApp(user, model.getApp()); + authManager.verifyAppHasTheModel(appGuid, model); + + val now = LocalDate.now(UTC); + val endDate = Date.from(now.atStartOfDay(UTC).toInstant()); + val startDate = Date.from(now.minusMonths(statisticMonths).atStartOfDay(UTC).toInstant()); + + return statisticRepository.findAllSummarizedByDay(guid, startDate, endDate); + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/service/ResetPasswordTokenService.java b/java/admin/src/main/java/com/exadel/frs/service/ResetPasswordTokenService.java new file mode 100644 index 0000000000..bcfcdde506 --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/service/ResetPasswordTokenService.java @@ -0,0 +1,110 @@ +package com.exadel.frs.service; + +import static java.lang.Boolean.parseBoolean; +import static java.lang.String.format; +import static java.time.LocalDateTime.now; +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.MILLIS; +import static liquibase.repackaged.org.apache.commons.text.StringSubstitutor.replace; +import com.exadel.frs.commonservice.entity.ResetPasswordToken; +import com.exadel.frs.commonservice.entity.User; +import com.exadel.frs.exception.InvalidResetPasswordTokenException; +import com.exadel.frs.exception.MailServerDisabledException; +import com.exadel.frs.helpers.EmailSender; +import com.exadel.frs.repository.ResetPasswordTokenRepository; +import java.util.Map; +import java.util.UUID; +import liquibase.repackaged.org.apache.commons.text.StringSubstitutor; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@Slf4j +@RequiredArgsConstructor +public class ResetPasswordTokenService { + + @Value("${forgot-password.reset-password-token.expires}") + private long tokenExpires; + + private final ResetPasswordTokenRepository tokenRepository; + private final UserService userService; + private final EmailSender emailSender; + private final Environment env; + + @Transactional + @Scheduled(cron = "${forgot-password.cleaner.scheduler.cron}", zone = "UTC") + public void deleteExpiredTokens() { + tokenRepository.deleteAllByExpiresInBefore(now(UTC)); + } + + @Transactional + public User exchangeTokenOnUser(final String rawToken) { + try { + val token = UUID.fromString(rawToken); + val resetPasswordToken = tokenRepository.findById(token) + .orElseThrow(InvalidResetPasswordTokenException::new); + + if (resetPasswordToken.getExpiresIn().isAfter(now(UTC))) { + tokenRepository.delete(resetPasswordToken); + return resetPasswordToken.getUser(); + } else { + throw new InvalidResetPasswordTokenException(); + } + } catch (Exception e) { + log.info(e.getMessage()); + throw new InvalidResetPasswordTokenException(); + } + } + + @Transactional + public void assignAndSendToken(final String email) { + val isMailServerEnabled = parseBoolean(env.getProperty("spring.mail.enable")); + + if (isMailServerEnabled) { + val user = userService.getEnabledUserByEmail(email); + val token = assignToken(user); + + sendToken(email, token); + } else { + log.warn("Cannot send an email due to the email server being disabled!"); + throw new MailServerDisabledException(); + } + } + + private UUID assignToken(final User user) { + val expiresIn = now(UTC).plus(tokenExpires, MILLIS); + val resetPasswordToken = new ResetPasswordToken(expiresIn, user); + + tokenRepository.deleteByUserEmail(user.getEmail()); + tokenRepository.flush(); + tokenRepository.save(resetPasswordToken); + + return resetPasswordToken.getToken(); + } + + private void sendToken(final String email, final UUID token) { + val messageParams = Map.of( + "host", env.getProperty("host.frs"), + "token", token.toString() + ); + + val message = StringSubstitutor.replace(""" + In order to reset a password click the link below:
+ + ${host}/reset-password?token=${token} + + """, messageParams, "${", "}"); + + emailSender.sendMail( + email, + "CompreFace Reset Password", + message + ); + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/service/StatisticService.java b/java/admin/src/main/java/com/exadel/frs/service/StatisticService.java new file mode 100644 index 0000000000..58d9bcc24c --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/service/StatisticService.java @@ -0,0 +1,105 @@ +package com.exadel.frs.service; + +import static com.exadel.frs.commonservice.enums.GlobalRole.OWNER; +import static org.apache.commons.lang3.Range.between; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import com.exadel.frs.commonservice.entity.InstallInfo; +import com.exadel.frs.commonservice.projection.ModelSubjectProjection; +import com.exadel.frs.commonservice.entity.User; +import com.exadel.frs.commonservice.repository.InstallInfoRepository; +import com.exadel.frs.commonservice.repository.ModelRepository; +import com.exadel.frs.commonservice.repository.UserRepository; +import com.exadel.frs.commonservice.system.feign.ApperyStatisticsClient; +import com.exadel.frs.commonservice.system.feign.StatisticsFacesEntity; +import feign.FeignException; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.Range; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class StatisticService { + + private static final List> RANGES = List.of( + between(1L, 10L), between(11L, 50L), between(51L, 200L), + between(201L, 500L), between(501L, 2000L), between(2001L, 10000L), + between(10001L, 50000L), between(50001L, 200000L), between(200001L, 1000000L) + ); + + @Value("${app.feign.appery-io.api-key}") + private String apperyApiKey; + + private final ApperyStatisticsClient apperyClient; + private final InstallInfoRepository installInfoRepository; + private final ModelRepository modelRepository; + private final UserRepository userRepository; + + @Scheduled(cron = "@midnight", zone = "UTC") + public void recordStatistics() { + boolean statisticsAreNotAllowed = !areStatisticsAllowed(); + if (statisticsAreNotAllowed) { + log.info("Statistics are not allowed"); + return; + } + + List subjectCountPerModel = modelRepository.getModelSubjectsCount(); + InstallInfo installInfo = installInfoRepository.findTopByOrderByInstallGuid(); + + if (installInfo == null) { + log.warn("In order to record statistics, at least one InstallInfo must be present"); + return; + } + + List statistics = createStatistics(installInfo.getInstallGuid(), subjectCountPerModel); + sendStatistics(statistics); + } + + private boolean areStatisticsAllowed() { + if (isNotBlank(apperyApiKey)) { + User owner = userRepository.findByGlobalRole(OWNER); + return owner != null && owner.isAllowStatistics(); + } else { + return false; + } + } + + private List createStatistics(String installInfoGuid, List subjectCountPerModel) { + return subjectCountPerModel.stream() + .map(subjectCount -> createStatistic(installInfoGuid, subjectCount)) + .toList(); + } + + private StatisticsFacesEntity createStatistic(String installInfoGuid, ModelSubjectProjection subjectCount) { + return new StatisticsFacesEntity( + installInfoGuid, + subjectCount.guid(), + getSubjectRange(subjectCount.subjectCount()) + ); + } + + private String getSubjectRange(Long subjectCount) { + if (subjectCount == null || subjectCount == 0) { + return "0"; + } + + return RANGES.stream() + .filter(range -> range.contains(subjectCount)) + .map(range -> range.getMinimum() + "-" + range.getMaximum()) + .findFirst() + .orElse("1000001+"); + } + + private void sendStatistics(List statistics) { + try { + statistics + .forEach(statistic -> apperyClient.create(apperyApiKey, statistic)); + } catch (FeignException e) { + log.info(e.getMessage()); + } + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/service/UserService.java b/java/admin/src/main/java/com/exadel/frs/service/UserService.java index 2b202b21f2..61563ba47c 100644 --- a/java/admin/src/main/java/com/exadel/frs/service/UserService.java +++ b/java/admin/src/main/java/com/exadel/frs/service/UserService.java @@ -25,14 +25,15 @@ import static org.apache.commons.lang3.BooleanUtils.isNotTrue; import static org.apache.commons.lang3.StringUtils.isBlank; import com.exadel.frs.commonservice.annotation.CollectStatistics; -import com.exadel.frs.commonservice.exception.EmptyRequiredFieldException; -import com.exadel.frs.dto.ui.UserCreateDto; -import com.exadel.frs.dto.ui.UserDeleteDto; -import com.exadel.frs.dto.ui.UserRoleUpdateDto; -import com.exadel.frs.dto.ui.UserUpdateDto; import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.enums.GlobalRole; import com.exadel.frs.commonservice.enums.Replacer; +import com.exadel.frs.commonservice.exception.EmptyRequiredFieldException; +import com.exadel.frs.commonservice.repository.UserRepository; +import com.exadel.frs.dto.UserCreateDto; +import com.exadel.frs.dto.UserDeleteDto; +import com.exadel.frs.dto.UserRoleUpdateDto; +import com.exadel.frs.dto.UserUpdateDto; import com.exadel.frs.exception.EmailAlreadyRegisteredException; import com.exadel.frs.exception.IncorrectUserPasswordException; import com.exadel.frs.exception.InsufficientPrivilegesException; @@ -41,12 +42,13 @@ import com.exadel.frs.exception.SelfRoleChangeException; import com.exadel.frs.exception.UserDoesNotExistException; import com.exadel.frs.helpers.EmailSender; -import com.exadel.frs.commonservice.repository.UserRepository; import com.exadel.frs.system.security.AuthorizationManager; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.UUID; +import liquibase.repackaged.org.apache.commons.text.StringSubstitutor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -125,14 +127,23 @@ public String generateRegistrationToken() { } private void sendRegistrationTokenToUser(final User user) { - val message = "Please, confirm your registration clicking the link below:\n" - + env.getProperty("host.frs") - + "/admin/user/registration/confirm?token=" - + user.getRegistrationToken(); - - val subject = "CompreFace Registration"; - - emailSender.sendMail(user.getEmail(), subject, message); + val messageParams = Map.of( + "host", env.getProperty("host.frs"), + "token", user.getRegistrationToken() + ); + + val message = StringSubstitutor.replace(""" + Please, confirm your registration clicking the link below:
+ + ${host}/admin/user/registration/confirm?token=${token} + + """, messageParams, "${", "}"); + + emailSender.sendMail( + user.getEmail(), + "CompreFace Registration", + message + ); } private void validateUserCreateDto(final UserCreateDto userCreateDto) { @@ -191,14 +202,21 @@ public void removeExpiredRegistrationTokens() { userRepository.deleteByEnabledFalseAndRegTimeBefore(seconds); } + @Transactional public void confirmRegistration(final String token) { val user = userRepository.findByRegistrationToken(token) .orElseThrow(RegistrationTokenExpiredException::new); - user.setEnabled(true); - user.setRegistrationToken(null); + synchronized (this) { + if (!userRepository.isOwnerPresent()) { + user.setGlobalRole(OWNER); + } - userRepository.save(user); + user.setEnabled(true); + user.setRegistrationToken(null); + + userRepository.flush(); + } } private void manageOwnedAppsByUserBeingDeleted(final UserDeleteDto userDeleteDto) { @@ -228,6 +246,7 @@ public User updateDemoUser(UserCreateDto userCreateDto) { user.setAllowStatistics(userCreateDto.isAllowStatistics()); if (isMailServerEnabled) { + user.setGlobalRole(USER); user.setRegistrationToken(generateRegistrationToken()); sendRegistrationTokenToUser(user); } @@ -317,4 +336,10 @@ public void changePassword(Long userId, String oldPwd, String newPwd) { userRepository.save(user); } -} \ No newline at end of file + + @Transactional + public void resetPassword(final User user, final String password) { + user.setPassword(encoder.encode(password)); + userRepository.save(user); + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/global/Constants.java b/java/admin/src/main/java/com/exadel/frs/system/global/Constants.java index b92d113d4e..c7ee5cca9c 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/global/Constants.java +++ b/java/admin/src/main/java/com/exadel/frs/system/global/Constants.java @@ -16,9 +16,14 @@ package com.exadel.frs.system.global; +import lombok.experimental.UtilityClass; + +@UtilityClass public class Constants { + public static final String ADMIN = "/admin"; public static final String GUID_EXAMPLE = "3913717b-a40b-4d6f-acc4-a861aa612651"; public static final String ACCESS_TOKEN_COOKIE_NAME = "CFSESSION"; + public static final String REFRESH_TOKEN_COOKIE_NAME = "REFRESH_TOKEN"; public static final String DEMO_GUID = "00000000-0000-0000-0000-000000000001"; -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/AuthenticationKeyGeneratorImpl.java b/java/admin/src/main/java/com/exadel/frs/system/security/AuthenticationKeyGeneratorImpl.java index 188ece5153..901613ad04 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/AuthenticationKeyGeneratorImpl.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/AuthenticationKeyGeneratorImpl.java @@ -32,13 +32,13 @@ public class AuthenticationKeyGeneratorImpl extends DefaultAuthenticationKeyGene private static final String USERNAME = "username"; public String extractKey(OAuth2Authentication authentication) { - Map values = new LinkedHashMap(); + Map values = new LinkedHashMap<>(); OAuth2Request authorizationRequest = authentication.getOAuth2Request(); values.put(USERNAME, authentication.getName()); values.put(CLIENT_ID, authorizationRequest.getClientId()); values.put(NO_CACHE_UUID, authorizationRequest.getExtensions().get(NO_CACHE_UUID).toString()); if (authorizationRequest.getScope() != null) { - values.put(SCOPE, OAuth2Utils.formatParameterList(new TreeSet(authorizationRequest.getScope()))); + values.put(SCOPE, OAuth2Utils.formatParameterList(new TreeSet<>(authorizationRequest.getScope()))); } return generateKey(values); } diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/AuthorizationManager.java b/java/admin/src/main/java/com/exadel/frs/system/security/AuthorizationManager.java index e1bd3ff3d0..29a87272ab 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/AuthorizationManager.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/AuthorizationManager.java @@ -19,7 +19,7 @@ import static com.exadel.frs.commonservice.enums.GlobalRole.ADMINISTRATOR; import static com.exadel.frs.commonservice.enums.GlobalRole.OWNER; import static com.exadel.frs.commonservice.enums.GlobalRole.USER; -import com.exadel.frs.dto.ui.UserDeleteDto; +import com.exadel.frs.dto.UserDeleteDto; import com.exadel.frs.commonservice.entity.App; import com.exadel.frs.commonservice.entity.Model; import com.exadel.frs.commonservice.entity.User; @@ -123,4 +123,4 @@ public void verifyCanDeleteUser(final UserDeleteDto userDeleteDto) { throw new InsufficientPrivilegesException(); } } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/CustomJdbcTokenStore.java b/java/admin/src/main/java/com/exadel/frs/system/security/CustomJdbcTokenStore.java new file mode 100644 index 0000000000..8e0ef53249 --- /dev/null +++ b/java/admin/src/main/java/com/exadel/frs/system/security/CustomJdbcTokenStore.java @@ -0,0 +1,90 @@ +package com.exadel.frs.system.security; + +import static java.time.ZoneOffset.UTC; +import java.sql.Types; +import java.time.LocalDateTime; +import javax.sql.DataSource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.support.SqlLobValue; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.security.oauth2.common.DefaultExpiringOAuth2RefreshToken; +import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.security.oauth2.common.OAuth2RefreshToken; +import org.springframework.security.oauth2.provider.OAuth2Authentication; +import org.springframework.security.oauth2.provider.token.AuthenticationKeyGenerator; +import org.springframework.security.oauth2.provider.token.store.JdbcTokenStore; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +@Slf4j +@Component +public class CustomJdbcTokenStore extends JdbcTokenStore { + + private static final String INSERT_ACCESS_TOKEN_WITH_EXPIRATION_SQL = "insert into oauth_access_token (token_id, token, authentication_id, user_name, client_id, authentication, refresh_token, expiration) values (?, ?, ?, ?, ?, ?, ?,?)"; + private static final String INSERT_REFRESH_TOKEN_WITH_EXPIRATION_SQL = "insert into oauth_refresh_token (token_id, token, authentication, expiration) values (?, ?, ?, ?)"; + private static final String REMOVE_EXPIRED_ACCESS_TOKENS_SQL = "delete from oauth_access_token where expiration < ?"; + private static final String REMOVE_EXPIRED_REFRESH_TOKENS_SQL = "delete from oauth_refresh_token where expiration < ?"; + + private final JdbcTemplate jdbcTemplate; + private final AuthenticationKeyGenerator authenticationKeyGenerator; + + public CustomJdbcTokenStore(DataSource dataSource) { + super(dataSource); + this.jdbcTemplate = new JdbcTemplate(dataSource); + this.authenticationKeyGenerator = new AuthenticationKeyGeneratorImpl(); + this.setAuthenticationKeyGenerator(this.authenticationKeyGenerator); + } + + @Override + public void storeAccessToken(OAuth2AccessToken token, OAuth2Authentication authentication) { + String refreshToken = null; + if (token.getRefreshToken() != null) { + refreshToken = token.getRefreshToken().getValue(); + } + + if (readAccessToken(token.getValue()) != null) { + removeAccessToken(token.getValue()); + } + + jdbcTemplate.update(INSERT_ACCESS_TOKEN_WITH_EXPIRATION_SQL, new Object[]{extractTokenKey(token.getValue()), + new SqlLobValue(serializeAccessToken(token)), this.authenticationKeyGenerator.extractKey(authentication), + authentication.isClientOnly() ? null : authentication.getName(), + authentication.getOAuth2Request().getClientId(), + new SqlLobValue(serializeAuthentication(authentication)), extractTokenKey(refreshToken), + token.getExpiration().toInstant().atOffset(UTC).toLocalDateTime()}, + new int[]{Types.VARCHAR, Types.BLOB, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.BLOB, Types.VARCHAR, Types.TIMESTAMP} + ); + } + + @Override + public void storeRefreshToken(OAuth2RefreshToken refreshToken, OAuth2Authentication authentication) { + DefaultExpiringOAuth2RefreshToken oAuth2RefreshToken = (DefaultExpiringOAuth2RefreshToken) refreshToken; + jdbcTemplate.update(INSERT_REFRESH_TOKEN_WITH_EXPIRATION_SQL, new Object[]{ + extractTokenKey(refreshToken.getValue()), + new SqlLobValue(serializeRefreshToken(refreshToken)), + new SqlLobValue(serializeAuthentication(authentication)), + oAuth2RefreshToken.getExpiration().toInstant().atOffset(UTC).toLocalDateTime()}, + new int[]{Types.VARCHAR, Types.BLOB, Types.BLOB, Types.TIMESTAMP} + ); + } + + @Transactional + @Scheduled(cron = "@weekly", zone = "UTC") + public void removeExpiredTokens() { + LocalDateTime now = LocalDateTime.now(UTC); + int accessTokenCount = this.jdbcTemplate.update( + REMOVE_EXPIRED_ACCESS_TOKENS_SQL, + now + ); + int refreshTokenCount = this.jdbcTemplate.update( + REMOVE_EXPIRED_REFRESH_TOKENS_SQL, + now + ); + log.info( + "Removed {} expired access tokens and {} expired update tokens", + accessTokenCount, + refreshTokenCount + ); + } +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/CustomOAuth2Exception.java b/java/admin/src/main/java/com/exadel/frs/system/security/CustomOAuth2Exception.java index dc209f3eaa..9d558fe9a6 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/CustomOAuth2Exception.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/CustomOAuth2Exception.java @@ -23,4 +23,4 @@ public class CustomOAuth2Exception extends OAuth2Exception { public CustomOAuth2Exception(String msg) { super(msg); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/CustomUserDetailsService.java b/java/admin/src/main/java/com/exadel/frs/system/security/CustomUserDetailsService.java index a7885dc2e9..b790593275 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/CustomUserDetailsService.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/CustomUserDetailsService.java @@ -38,4 +38,4 @@ public UserDetails loadUserByUsername(String email) throws UsernameNotFoundExcep String.format("User %s does not exists", email)) ); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/TokenServicesImpl.java b/java/admin/src/main/java/com/exadel/frs/system/security/TokenServicesImpl.java index 875a8054e7..c2f5b5fe02 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/TokenServicesImpl.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/TokenServicesImpl.java @@ -46,8 +46,7 @@ public OAuth2AccessToken createAccessToken(OAuth2Authentication authentication) Collection oAuth2AccessTokens = tokenStore.findTokensByClientIdAndUserName(authentication .getOAuth2Request().getClientId(), authentication.getName()); oAuth2AccessTokens.forEach(accessToken -> { - if (accessToken.getRefreshToken() instanceof ExpiringOAuth2RefreshToken) { - ExpiringOAuth2RefreshToken expiring = (ExpiringOAuth2RefreshToken) accessToken.getRefreshToken(); + if (accessToken.getRefreshToken() instanceof ExpiringOAuth2RefreshToken expiring) { if (System.currentTimeMillis() > expiring.getExpiration().getTime()) { tokenStore.removeAccessToken(accessToken); tokenStore.removeRefreshToken(accessToken.getRefreshToken()); @@ -56,4 +55,4 @@ public OAuth2AccessToken createAccessToken(OAuth2Authentication authentication) }); return super.createAccessToken(authentication); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/client/Client.java b/java/admin/src/main/java/com/exadel/frs/system/security/client/Client.java index 4499ba3220..72827f8093 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/client/Client.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/client/Client.java @@ -137,4 +137,4 @@ public boolean isAutoApprove(String scope) { public Map getAdditionalInformation() { return new HashMap<>(); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/client/ClientRepository.java b/java/admin/src/main/java/com/exadel/frs/system/security/client/ClientRepository.java index edf53af0e2..6911bc1898 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/client/ClientRepository.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/client/ClientRepository.java @@ -24,4 +24,4 @@ public interface ClientRepository extends JpaRepository { Optional findByClientId(String clientId); -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/client/ClientService.java b/java/admin/src/main/java/com/exadel/frs/system/security/client/ClientService.java index d7ac2e020d..977bc170e5 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/client/ClientService.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/client/ClientService.java @@ -40,4 +40,4 @@ public ClientDetails loadClientByClientId(String clientId) { public List saveAll(List clientsDetail) { return clientRepository.saveAll(clientsDetail); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/client/OAuthClientProperties.java b/java/admin/src/main/java/com/exadel/frs/system/security/client/OAuthClientProperties.java index f63e72fa89..2f42a95b6c 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/client/OAuthClientProperties.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/client/OAuthClientProperties.java @@ -54,4 +54,4 @@ public static final class Client { private Integer refreshTokenValidity; private List resourceIds; } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/config/AuthServerConfig.java b/java/admin/src/main/java/com/exadel/frs/system/security/config/AuthServerConfig.java index 86772004e7..d5ee54abcb 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/config/AuthServerConfig.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/config/AuthServerConfig.java @@ -16,8 +16,7 @@ package com.exadel.frs.system.security.config; -import static java.util.stream.Collectors.toList; -import com.exadel.frs.system.security.AuthenticationKeyGeneratorImpl; +import static com.exadel.frs.system.global.Constants.ADMIN; import com.exadel.frs.system.security.CustomOAuth2Exception; import com.exadel.frs.system.security.CustomUserDetailsService; import com.exadel.frs.system.security.TokenServicesImpl; @@ -25,7 +24,6 @@ import com.exadel.frs.system.security.client.ClientService; import com.exadel.frs.system.security.client.OAuthClientProperties; import com.exadel.frs.system.security.endpoint.CustomTokenEndpoint; -import javax.sql.DataSource; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -55,16 +53,9 @@ public class AuthServerConfig extends AuthorizationServerConfigurerAdapter { private final AuthenticationManager authenticationManager; private final ClientService clientService; private final CustomUserDetailsService userDetailsService; - private final DataSource dataSource; private final PasswordEncoder passwordEncoder; private final OAuthClientProperties authClientProperties; - - @Bean - public JdbcTokenStore tokenStore() { - JdbcTokenStore tokenStore = new JdbcTokenStore(dataSource); - tokenStore.setAuthenticationKeyGenerator(new AuthenticationKeyGeneratorImpl()); - return tokenStore; - } + private final JdbcTokenStore tokenStore; @Bean @Primary @@ -81,7 +72,7 @@ public TokenEndpoint tokenEndpoint(AuthorizationServerEndpointsConfiguration con @Bean public DefaultTokenServices tokenServices() { - TokenServicesImpl tokenServices = new TokenServicesImpl(tokenStore()); + TokenServicesImpl tokenServices = new TokenServicesImpl(tokenStore); tokenServices.setClientDetailsService(clientService); return tokenServices; } @@ -110,21 +101,21 @@ public void configure(final ClientDetailsServiceConfigurer clients) throws Excep .setAccessTokenValidity(it.getAccessTokenValidity()) .setRefreshTokenValidity(it.getRefreshTokenValidity()) .setAutoApprove("*")) - .collect(toList()); + .toList(); clientService.saveAll(appClients); } @Override public void configure(AuthorizationServerEndpointsConfigurer endpoints) { endpoints - .tokenStore(tokenStore()) + .tokenStore(tokenStore) .tokenServices(tokenServices()) .authenticationManager(authenticationManager) - .userDetailsService(userDetailsService); + .userDetailsService(userDetailsService) + .pathMapping("/oauth/token", ADMIN + "/oauth/token"); endpoints.exceptionTranslator(exception -> { - if (exception instanceof OAuth2Exception) { - OAuth2Exception oAuth2Exception = (OAuth2Exception) exception; + if (exception instanceof OAuth2Exception oAuth2Exception) { return ResponseEntity .status(oAuth2Exception.getHttpErrorCode()) .body(new CustomOAuth2Exception(oAuth2Exception.getMessage())); @@ -133,4 +124,4 @@ public void configure(AuthorizationServerEndpointsConfigurer endpoints) { } }); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/config/ResourceServerConfig.java b/java/admin/src/main/java/com/exadel/frs/system/security/config/ResourceServerConfig.java index e76f7a2c33..316d35ea95 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/config/ResourceServerConfig.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/config/ResourceServerConfig.java @@ -16,6 +16,7 @@ package com.exadel.frs.system.security.config; +import static com.exadel.frs.system.global.Constants.ADMIN; import com.exadel.frs.system.security.CookieTokenExtractor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; @@ -40,7 +41,8 @@ public void configure(HttpSecurity http) throws Exception { http.csrf() .disable() .authorizeRequests() - .antMatchers("/actuator/**", "/user/register", "/user/registration/confirm", "/user/demo/model", "/api/**").permitAll() + .antMatchers("/actuator/**", ADMIN + "/user/register", ADMIN + "/user/forgot-password", ADMIN + "/user/reset-password", + ADMIN + "/user/registration/confirm", ADMIN + "/user/demo/model", "/api/**", ADMIN + "/status", ADMIN + "/config").permitAll() .anyRequest().authenticated() .and().sessionManagement().sessionCreationPolicy(SessionCreationPolicy.STATELESS); } @@ -52,4 +54,4 @@ public void configure(ResourceServerSecurityConfigurer resources) { .tokenStore(jdbcTokenStore) .tokenExtractor(tokenExtractor); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/config/WebSecurityConfig.java b/java/admin/src/main/java/com/exadel/frs/system/security/config/WebSecurityConfig.java index d6c44c6d45..57f91aa3b0 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/config/WebSecurityConfig.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/config/WebSecurityConfig.java @@ -16,6 +16,7 @@ package com.exadel.frs.system.security.config; +import static com.exadel.frs.system.global.Constants.ADMIN; import com.exadel.frs.system.security.CustomUserDetailsService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; @@ -51,8 +52,9 @@ protected void configure(HttpSecurity http) throws Exception { .sessionManagement().sessionCreationPolicy(SessionCreationPolicy.STATELESS) .and() .authorizeRequests() - .antMatchers(HttpMethod.OPTIONS, "/oauth/**").permitAll() - .antMatchers("/actuator/**", "/user/register", "/api/**").permitAll() + .antMatchers(HttpMethod.OPTIONS, ADMIN + "/oauth/**").permitAll() + .antMatchers("/actuator/**", ADMIN + "/user/register", ADMIN + "/user/forgot-password", ADMIN + "/user/reset-password", ADMIN + "/config", + "/api/**").permitAll() .anyRequest().authenticated() .and() .logout() @@ -76,4 +78,4 @@ public void configure(WebSecurity web) { "/swagger-ui**", "/webjars/**", "/lms/**" ); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/endpoint/CustomTokenEndpoint.java b/java/admin/src/main/java/com/exadel/frs/system/security/endpoint/CustomTokenEndpoint.java index 2c58967728..a05f8cef1d 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/endpoint/CustomTokenEndpoint.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/endpoint/CustomTokenEndpoint.java @@ -1,48 +1,114 @@ package com.exadel.frs.system.security.endpoint; import static com.exadel.frs.system.global.Constants.ACCESS_TOKEN_COOKIE_NAME; +import static com.exadel.frs.system.global.Constants.ADMIN; +import static com.exadel.frs.system.global.Constants.REFRESH_TOKEN_COOKIE_NAME; +import static org.apache.commons.lang3.StringUtils.EMPTY; import com.exadel.frs.commonservice.entity.User; import java.security.Principal; +import java.util.Arrays; import java.util.Map; +import java.util.Objects; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; import lombok.val; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseCookie; import org.springframework.http.ResponseEntity; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.oauth2.common.DefaultExpiringOAuth2RefreshToken; import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.security.oauth2.common.OAuth2RefreshToken; import org.springframework.security.oauth2.provider.endpoint.TokenEndpoint; import org.springframework.web.HttpRequestMethodNotSupportedException; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -@RequestMapping(value = "/oauth/token") +@RequestMapping(ADMIN + "/oauth/token") public class CustomTokenEndpoint extends TokenEndpoint { + @Autowired + private HttpServletRequest currentRequest; + + @Override @PostMapping public ResponseEntity postAccessToken( Principal principal, @RequestParam Map parameters ) throws HttpRequestMethodNotSupportedException { - if (principal instanceof UsernamePasswordAuthenticationToken) { - if (((UsernamePasswordAuthenticationToken) principal).getPrincipal() instanceof User) { - return ResponseEntity.status(HttpStatus.OK).build(); - } + if (principal instanceof UsernamePasswordAuthenticationToken authenticationToken + && authenticationToken.getPrincipal() instanceof User) { + return ResponseEntity.status(HttpStatus.OK).build(); + } + + if (isRefreshTokenGrantType(parameters)) { + val refreshTokenValue = extractRefreshTokenCookieValueFromRequest(currentRequest); + parameters.put("refresh_token", refreshTokenValue); } - val defaultResponse = super.postAccessToken(principal, parameters); - val defaultToken = defaultResponse.getBody(); + val tokenResponse = super.postAccessToken(principal, parameters); + + val accessToken = tokenResponse.getBody(); + val refreshToken = accessToken.getRefreshToken(); + + val accessTokenCookie = buildAccessTokenCookie(accessToken); + val refreshTokenCookie = buildRefreshTokenCookie(refreshToken); - val cookie = ResponseCookie.from(ACCESS_TOKEN_COOKIE_NAME, defaultToken.getValue()) - .httpOnly(true) - .maxAge(defaultToken.getExpiresIn()) - .path("/admin") - .build(); val headers = new HttpHeaders(); - headers.add(HttpHeaders.SET_COOKIE, cookie.toString()); + headers.add(HttpHeaders.SET_COOKIE, accessTokenCookie); + headers.add(HttpHeaders.SET_COOKIE, refreshTokenCookie); + + return ResponseEntity.status(HttpStatus.OK).headers(headers).body(accessToken); + } + + private String extractRefreshTokenCookieValueFromRequest(final HttpServletRequest request) { + val cookies = Objects.requireNonNullElse(request.getCookies(), new Cookie[0]); + + return Arrays.stream(cookies) + .filter(cookie -> cookie.getName().equals(REFRESH_TOKEN_COOKIE_NAME)) + .findFirst() + .map(Cookie::getValue) + .orElse(EMPTY); + } + + private boolean isRefreshTokenGrantType(final Map requestParams) { + return "refresh_token".equals(requestParams.get("grant_type")); + } + + private String buildAccessTokenCookie(final OAuth2AccessToken token) { + val value = token.getValue(); + val expiresIn = token.getExpiresIn(); + + return ResponseCookie.from(ACCESS_TOKEN_COOKIE_NAME, value) + .httpOnly(true) + .maxAge(expiresIn) + .path("/admin") + .build() + .toString(); + } + + private String buildRefreshTokenCookie(final OAuth2RefreshToken token) { + val value = token.getValue(); + val expiresIn = extractExpiresInFromRefreshToken(token); + + return ResponseCookie.from(REFRESH_TOKEN_COOKIE_NAME, value) + .httpOnly(true) + .maxAge(expiresIn) + .path("/admin/oauth/token") + .build() + .toString(); + } + + private long extractExpiresInFromRefreshToken(final OAuth2RefreshToken token) { + val refreshToken = (DefaultExpiringOAuth2RefreshToken) token; + val expiration = refreshToken.getExpiration(); - return ResponseEntity.status(HttpStatus.OK).headers(headers).build(); + return expiration != null + ? (expiration.getTime() - System.currentTimeMillis()) / 1000L + : -1L; } } diff --git a/java/admin/src/main/java/com/exadel/frs/system/security/endpoint/RevokeTokenEndpoint.java b/java/admin/src/main/java/com/exadel/frs/system/security/endpoint/RevokeTokenEndpoint.java index db2fc37501..a590319a76 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/security/endpoint/RevokeTokenEndpoint.java +++ b/java/admin/src/main/java/com/exadel/frs/system/security/endpoint/RevokeTokenEndpoint.java @@ -1,43 +1,66 @@ package com.exadel.frs.system.security.endpoint; import static com.exadel.frs.system.global.Constants.ACCESS_TOKEN_COOKIE_NAME; +import static com.exadel.frs.system.global.Constants.ADMIN; +import static com.exadel.frs.system.global.Constants.REFRESH_TOKEN_COOKIE_NAME; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import java.util.Optional; +import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; +import lombok.RequiredArgsConstructor; import lombok.val; -import org.springframework.beans.factory.annotation.Autowired; +import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseCookie; import org.springframework.http.ResponseEntity; import org.springframework.security.oauth2.provider.endpoint.FrameworkEndpoint; import org.springframework.security.oauth2.provider.token.DefaultTokenServices; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.util.WebUtils; @FrameworkEndpoint +@RequiredArgsConstructor public class RevokeTokenEndpoint { - @Autowired - private DefaultTokenServices tokenServices; + private final DefaultTokenServices tokenServices; - @RequestMapping(method = RequestMethod.DELETE, value = "/oauth/token") @ResponseBody - public ResponseEntity revokeToken(HttpServletRequest request) { - if (WebUtils.getCookie(request, ACCESS_TOKEN_COOKIE_NAME) == null) { + @DeleteMapping(ADMIN + "/oauth/token") + public ResponseEntity revokeToken(HttpServletRequest request) { + if (isAccessTokenBlank(request)) { return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); } - val token = WebUtils.getCookie(request, ACCESS_TOKEN_COOKIE_NAME).getValue(); - tokenServices.revokeToken(token); - - val cookie = ResponseCookie.from(ACCESS_TOKEN_COOKIE_NAME, "") - .httpOnly(true) - .maxAge(0) - .path("/admin") - .build(); + + val accessToken = WebUtils.getCookie(request, ACCESS_TOKEN_COOKIE_NAME).getValue(); + tokenServices.revokeToken(accessToken); + + val accessTokenCookie = buildEmptyCookie(ACCESS_TOKEN_COOKIE_NAME, "/admin"); + val refreshTokenCookie = buildEmptyCookie(REFRESH_TOKEN_COOKIE_NAME, "/admin/oauth/token"); + val headers = new HttpHeaders(); - headers.add(HttpHeaders.SET_COOKIE, cookie.toString()); + headers.add(HttpHeaders.SET_COOKIE, accessTokenCookie); + headers.add(HttpHeaders.SET_COOKIE, refreshTokenCookie); return ResponseEntity.status(HttpStatus.OK).headers(headers).build(); } + + private String buildEmptyCookie(String name, String path) { + return ResponseCookie.from(name, EMPTY) + .httpOnly(true) + .maxAge(0) + .path(path) + .build() + .toString(); + } + + private boolean isAccessTokenBlank(HttpServletRequest request) { + val cookie = WebUtils.getCookie(request, ACCESS_TOKEN_COOKIE_NAME); + val cookieValue = Optional.ofNullable(cookie) + .map(Cookie::getValue) + .orElse(EMPTY); + + return StringUtils.isBlank(cookieValue); + } } diff --git a/java/admin/src/main/java/com/exadel/frs/system/swagger/SpringFoxConfig.java b/java/admin/src/main/java/com/exadel/frs/system/swagger/SpringFoxConfig.java index 91e8883653..2dba571911 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/swagger/SpringFoxConfig.java +++ b/java/admin/src/main/java/com/exadel/frs/system/swagger/SpringFoxConfig.java @@ -16,6 +16,7 @@ package com.exadel.frs.system.swagger; +import static com.exadel.frs.system.global.Constants.ADMIN; import static com.exadel.frs.system.security.client.OAuthClientProperties.ClientType.COMMON; import com.exadel.frs.system.security.client.OAuthClientProperties; import com.google.common.base.Predicates; @@ -70,7 +71,7 @@ private OAuth securitySchema() { val authorizationScopeList = new ArrayList(); authorizationScopeList.add(new AuthorizationScope("read write ", "all")); val grantTypes = new ArrayList(); - val creGrant = new ResourceOwnerPasswordCredentialsGrant(authLink + "/oauth/token"); + val creGrant = new ResourceOwnerPasswordCredentialsGrant(authLink + ADMIN + "/oauth/token"); grantTypes.add(creGrant); return new OAuth("oauth2schema", authorizationScopeList, grantTypes); @@ -101,4 +102,4 @@ public SecurityConfiguration securityInfo(OAuthClientProperties oAuthClientPrope .clientSecret(clientSecret) .build(); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/system/swagger/SwaggerInfoProperties.java b/java/admin/src/main/java/com/exadel/frs/system/swagger/SwaggerInfoProperties.java index d26407362a..b4702f8b0a 100644 --- a/java/admin/src/main/java/com/exadel/frs/system/swagger/SwaggerInfoProperties.java +++ b/java/admin/src/main/java/com/exadel/frs/system/swagger/SwaggerInfoProperties.java @@ -16,7 +16,7 @@ package com.exadel.frs.system.swagger; -import static org.springframework.util.StringUtils.isEmpty; +import static liquibase.repackaged.org.apache.commons.lang3.StringUtils.isNotEmpty; import lombok.Data; import lombok.val; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -46,27 +46,27 @@ public ApiInfo getApiInfo() { val builder = new ApiInfoBuilder(); - if (!isEmpty(this.contactName) - || !isEmpty(this.contactUrl) - || !isEmpty(this.contactEmail)) { + if (isNotEmpty(this.contactName) + || isNotEmpty(this.contactUrl) + || isNotEmpty(this.contactEmail)) { builder.contact(new Contact(this.contactName, this.contactUrl, this.contactEmail)); } - if (!isEmpty(this.description)) { + if (isNotEmpty(this.description)) { builder.description(this.description); } - if (!isEmpty(this.termsOfServiceUrl)) { + if (isNotEmpty(this.termsOfServiceUrl)) { builder.termsOfServiceUrl(this.termsOfServiceUrl); } - if (!isEmpty(this.title)) { + if (isNotEmpty(this.title)) { builder.title(this.title); } - if (!isEmpty(this.license)) { + if (isNotEmpty(this.license)) { builder.license(this.license); } - if (!isEmpty(this.version)) { + if (isNotEmpty(this.version)) { builder.version(this.version); } return builder.build(); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/validation/EmailValidator.java b/java/admin/src/main/java/com/exadel/frs/validation/EmailValidator.java index 978563a559..712a9b5f2d 100644 --- a/java/admin/src/main/java/com/exadel/frs/validation/EmailValidator.java +++ b/java/admin/src/main/java/com/exadel/frs/validation/EmailValidator.java @@ -17,7 +17,9 @@ package com.exadel.frs.validation; import java.util.regex.Pattern; +import lombok.experimental.UtilityClass; +@UtilityClass public class EmailValidator { private static final Pattern EMAIL_PATTERN = @@ -30,4 +32,4 @@ public static boolean isValid(final String email) { public static boolean isInvalid(final String email) { return !isValid(email); } -} \ No newline at end of file +} diff --git a/java/admin/src/main/java/com/exadel/frs/validation/ValidEnum.java b/java/admin/src/main/java/com/exadel/frs/validation/ValidEnum.java index 697067464b..6511b9136d 100644 --- a/java/admin/src/main/java/com/exadel/frs/validation/ValidEnum.java +++ b/java/admin/src/main/java/com/exadel/frs/validation/ValidEnum.java @@ -35,4 +35,4 @@ Class[] payload() default {}; Class> targetClassType(); -} \ No newline at end of file +} diff --git a/java/admin/src/main/resources/application.yml b/java/admin/src/main/resources/application.yml index 372656740a..7b07a2c377 100644 --- a/java/admin/src/main/resources/application.yml +++ b/java/admin/src/main/resources/application.yml @@ -1,5 +1,8 @@ server: port: ${CRUD_PORT:8080} + tomcat: + relaxed-path-chars: [ '[', ']' ] + relaxed-query-chars: [ '[', ']' ] app: security: @@ -18,19 +21,25 @@ app: appery-io: url: https://api.appery.io/rest/1/db/collections api-key: ${APPERY_API_KEY:#{null}} + faces: + connect-timeout: ${CONNECTION_TIMEOUT:10000} + read-timeout: ${READ_TIMEOUT:60000} + retryer: + max-attempts: ${MAX_ATTEMPTS:1} spring: profiles: active: prod servlet: multipart: - max-file-size: 10MB - max-request-size: 10MB + max-file-size: ${MAX_FILE_SIZE:5MB} + max-request-size: ${MAX_REQUEST_SIZE:10MB} datasource: driver-class-name: org.postgresql.Driver url: ${POSTGRES_URL:jdbc:postgresql://compreface-postgres-db:5432/frs} username: ${POSTGRES_USER:postgres} password: ${POSTGRES_PASSWORD:postgres} + continue-on-error: true hikari: maximum-pool-size: 3 minimum-idle: 3 @@ -46,6 +55,13 @@ spring: database: postgresql open-in-view: true generate-ddl: false + liquibase: + parameters: + common-client: + client-id: ${app.security.oauth2.clients.COMMON.client-id} + access-token-validity: ${app.security.oauth2.clients.COMMON.access-token-validity} + refresh-token-validity: ${app.security.oauth2.clients.COMMON.refresh-token-validity} + authorized-grant-types: ${app.security.oauth2.clients.COMMON.authorized-grant-types} mail: enable: ${ENABLE_EMAIL_SERVER:false} host: ${EMAIL_HOST:example.com} @@ -66,20 +82,6 @@ spring: auth: true main: allow-bean-definition-overriding: true - quartz: - job-store-type: jdbc - jdbc: - initialize-schema: always - properties: - org.quartz: - scheduler: - instanceName: statistics scheduler - instanceId: AUTO - jobStore: - driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate - threadPool: - # thread pool capacity (default is 10) - threadCount: 1 security: signing-key: MaYzkSjmkzPC57L @@ -106,6 +108,13 @@ registration: scheduler: period: 300000 +forgot-password: + reset-password-token: + expires: 900000 + cleaner: + scheduler: + cron: '0 0 0 * * ?' + # "environment" and "image" blocks should be same in those files: # * api/src/main/resources/application.properties # * admin/src/main/resources/application.properties @@ -123,7 +132,11 @@ image: - webp saveImagesToDB: ${SAVE_IMAGES_TO_DB:true} +statistic: + model: + months: ${MODEL_STATISTIC_MONTHS:6} + environment: servers: PYTHON: - url: ${PYTHON_URL:http://compreface-core:3000} \ No newline at end of file + url: ${PYTHON_URL:http://compreface-core:3000} diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.1.8.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.1.8.yaml new file mode 100644 index 0000000000..eb5f0614a8 --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.1.8.yaml @@ -0,0 +1,52 @@ +databaseChangeLog: + - changeSet: + id: add-detection-verification-services + author: Shreyansh Sancheti + preConditions: + - onFail: MARK_RAN + - sqlCheck: + expectedResult: 1 + sql: select count(1) from app where id=0 + changes: + - insert: + tableName: model + columns: + - column: + name: id + value: 9223372036854775807 + - column: + name: name + value: Detection_Demo + - column: + name: type + value: D + - column: + name: guid + value: 00000000-0000-0000-0000-000000000003 + - column: + name: app_id + value: 0 + - column: + name: api_key + value: 00000000-0000-0000-0000-000000000003 + - insert: + tableName: model + columns: + - column: + name: id + value: 9223372036854775806 + - column: + name: name + value: Verification_Demo + - column: + name: type + value: V + - column: + name: guid + value: 00000000-0000-0000-0000-000000000004 + - column: + name: app_id + value: 0 + - column: + name: api_key + value: 00000000-0000-0000-0000-000000000004 \ No newline at end of file diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.1.9.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.1.9.yaml new file mode 100644 index 0000000000..60669c5a30 --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.1.9.yaml @@ -0,0 +1,18 @@ +databaseChangeLog: + - changeSet: + id: add-created_date-field-to-model-entity + author: Khasan Sidikov + changes: + - addColumn: + tableName: model + columns: + - column: + name: created_date + type: timestamp + - changeSet: + id: update-model-table-created_date-column + author: Khasan Sidikov + changes: + - sql: + comment: Update created_date to now() + sql: UPDATE model SET created_date=now() WHERE created_date IS NULL; \ No newline at end of file diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.2.0.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.0.yaml new file mode 100644 index 0000000000..df2a79461a --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.0.yaml @@ -0,0 +1,50 @@ +databaseChangeLog: + - changeSet: + id: create-model-statistic-table + author: Volodymyr Bushko + changes: + # model_statistic + - createTable: + tableName: model_statistic + columns: + - column: + name: id + type: bigint + - column: + name: request_count + type: int + - column: + name: model_id + type: bigint + - column: + name: created_date + type: timestamp + + - addPrimaryKey: + columnNames: id + constraintName: pk_model_statistic + tableName: model_statistic + + - addForeignKeyConstraint: + baseColumnNames: model_id + baseTableName: model_statistic + referencedColumnNames: id + referencedTableName: model + constraintName: fk_model_id + onDelete: CASCADE + onUpdate: CASCADE + + - addNotNullConstraint: + tableName: model_statistic + columnName: request_count + + - addNotNullConstraint: + tableName: model_statistic + columnName: model_id + + - addNotNullConstraint: + tableName: model_statistic + columnName: created_date + + - createSequence: + sequenceName: model_statistic_id_seq diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.2.1.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.1.yaml new file mode 100644 index 0000000000..d310e97571 --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.1.yaml @@ -0,0 +1,43 @@ +databaseChangeLog: + - changeSet: + id: create-table-lock-table + author: Volodymyr Bushko + changes: + # table_lock + - createTable: + tableName: table_lock + columns: + - column: + name: id + type: uuid + - column: + name: lock_name + type: varchar(50) + + - addPrimaryKey: + columnNames: id + constraintName: pk_table_lock + tableName: table_lock + + - addNotNullConstraint: + tableName: table_lock + columnName: lock_name + + - addUniqueConstraint: + columnNames: lock_name + constraintName: table_lock_lock_name_uindex + tableName: table_lock + + - changeSet: + id: insert-into-table-lock-model_statistic_lock-row + author: Volodymyr Bushko + changes: + - insert: + tableName: table_lock + columns: + - column: + name: id + value: 00000000-0000-0000-0000-000000000001 + - column: + name: lock_name + value: MODEL_STATISTIC_LOCK diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.2.2.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.2.yaml new file mode 100644 index 0000000000..28cc025914 --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.2.yaml @@ -0,0 +1,13 @@ +databaseChangeLog: + - changeSet: + id: drop-app_model-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: app_model + changes: + - dropTable: + schemaName: public + tableName: app_model diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.2.3.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.3.yaml new file mode 100644 index 0000000000..517806c34f --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.3.yaml @@ -0,0 +1,45 @@ +databaseChangeLog: + - changeSet: + id: create-reset_password_token-table + author: Volodymyr Bushko + changes: + # reset_password_token + - createTable: + tableName: reset_password_token + columns: + - column: + name: token + type: uuid + - column: + name: user_email + type: varchar(63) + - column: + name: expires_in + type: timestamp + + - addPrimaryKey: + columnNames: token + tableName: reset_password_token + constraintName: pk_reset_password_token + + - addForeignKeyConstraint: + baseColumnNames: user_email + baseTableName: reset_password_token + referencedColumnNames: email + referencedTableName: user + constraintName: fk_reset_password_token_user_email + onDelete: CASCADE + onUpdate: CASCADE + + - addNotNullConstraint: + columnName: user_email + tableName: reset_password_token + + - addNotNullConstraint: + columnName: expires_in + tableName: reset_password_token + + - addUniqueConstraint: + columnNames: user_email + tableName: reset_password_token + constraintName: reset_password_token_user_email_uindex diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.2.4.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.4.yaml new file mode 100644 index 0000000000..959452820e --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.4.yaml @@ -0,0 +1,23 @@ +databaseChangeLog: + - changeSet: + id: remove-special-characters-from-names + author: Volodymyr Bushko + changes: + - customChange: { + "class": "com.exadel.frs.commonservice.system.liquibase.customchange.RemoveSpecialCharactersCustomChange", + "table": "app", + "primaryKeyColumn": "id", + "targetColumn": "name" + } + - customChange: { + "class": "com.exadel.frs.commonservice.system.liquibase.customchange.RemoveSpecialCharactersCustomChange", + "table": "model", + "primaryKeyColumn": "id", + "targetColumn": "name" + } + - customChange: { + "class": "com.exadel.frs.commonservice.system.liquibase.customchange.RemoveSpecialCharactersCustomChange", + "table": "subject", + "primaryKeyColumn": "id", + "targetColumn": "subject_name" + } diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.2.5.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.5.yaml new file mode 100644 index 0000000000..04b8b0ddb9 --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.5.yaml @@ -0,0 +1,33 @@ +databaseChangeLog: + - changeSet: + id: expand-oauth_access_token-and-oauth_refresh_token-with-expiration-column + author: Volodymyr Bushko + changes: + # add an expiration column to the oauth_access_token & oauth_refresh_token tables + - addColumn: + tableName: oauth_access_token + columns: + - column: + name: expiration + type: timestamp + - addColumn: + tableName: oauth_refresh_token + columns: + - column: + name: expiration + type: timestamp + # set the expiration columns in order to avoid conflicts + - customChange: { + "class": "com.exadel.frs.commonservice.system.liquibase.customchange.SetOAuthTokenExpirationCustomChange", + "clientId": "${common-client.client-id}", + "accessTokenValidity": "${common-client.access-token-validity}", + "refreshTokenValidity": "${common-client.refresh-token-validity}", + "authorizedGrantTypes": "${common-client.authorized-grant-types}" + } + # add a not-null constraint to the expiration columns + - addNotNullConstraint: + tableName: oauth_access_token + columnName: expiration + - addNotNullConstraint: + tableName: oauth_refresh_token + columnName: expiration diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.2.6.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.6.yaml new file mode 100644 index 0000000000..5afab9bacc --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.6.yaml @@ -0,0 +1,25 @@ +databaseChangeLog: + - changeSet: + id: drop-image-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: image + changes: + - dropTable: + schemaName: public + tableName: image + - changeSet: + id: drop-face-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: face + changes: + - dropTable: + schemaName: public + tableName: face diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-0.2.7.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.7.yaml new file mode 100644 index 0000000000..840f0ac15b --- /dev/null +++ b/java/admin/src/main/resources/db/changelog/db.changelog-0.2.7.yaml @@ -0,0 +1,144 @@ +databaseChangeLog: + - changeSet: + id: drop-qrtz_blob_triggers_table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_blob_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_blob_triggers + - changeSet: + id: drop-qrtz_calendars-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_calendars + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_calendars + - changeSet: + id: drop-qrtz_cron_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_cron_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_cron_triggers + - changeSet: + id: drop-qrtz_fired_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_fired_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_fired_triggers + - changeSet: + id: drop-qrtz_job_details-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_job_details + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_job_details + - changeSet: + id: drop-qrtz_locks-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_locks + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_locks + - changeSet: + id: drop-qrtz_paused_trigger_grps-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_paused_trigger_grps + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_paused_trigger_grps + - changeSet: + id: drop-qrtz_scheduler_state-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_scheduler_state + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_scheduler_state + - changeSet: + id: drop-qrtz_simple_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_simple_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_simple_triggers + - changeSet: + id: drop-qrtz_simprop_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_simprop_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_simprop_triggers + - changeSet: + id: drop-qrtz_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_triggers diff --git a/java/admin/src/main/resources/db/changelog/db.changelog-master.yaml b/java/admin/src/main/resources/db/changelog/db.changelog-master.yaml index 0ed46f2b67..d7dad0ef30 100644 --- a/java/admin/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/java/admin/src/main/resources/db/changelog/db.changelog-master.yaml @@ -33,3 +33,23 @@ databaseChangeLog: file: db/changelog/db.changelog-0.1.6.yaml - include: file: db/changelog/db.changelog-0.1.7.yaml + - include: + file: db/changelog/db.changelog-0.1.8.yaml + - include: + file: db/changelog/db.changelog-0.1.9.yaml + - include: + file: db/changelog/db.changelog-0.2.0.yaml + - include: + file: db/changelog/db.changelog-0.2.1.yaml + - include: + file: db/changelog/db.changelog-0.2.2.yaml + - include: + file: db/changelog/db.changelog-0.2.3.yaml + - include: + file: db/changelog/db.changelog-0.2.4.yaml + - include: + file: db/changelog/db.changelog-0.2.5.yaml + - include: + file: db/changelog/db.changelog-0.2.6.yaml + - include: + file: db/changelog/db.changelog-0.2.7.yaml diff --git a/java/admin/src/test/java/com/exadel/frs/DbHelper.java b/java/admin/src/test/java/com/exadel/frs/DbHelper.java index 9f0a6de5ad..e9424aa0a5 100644 --- a/java/admin/src/test/java/com/exadel/frs/DbHelper.java +++ b/java/admin/src/test/java/com/exadel/frs/DbHelper.java @@ -1,26 +1,46 @@ package com.exadel.frs; +import static com.exadel.frs.ItemsBuilder.makeApp; +import static com.exadel.frs.ItemsBuilder.makeEmbedding; +import static com.exadel.frs.ItemsBuilder.makeImg; +import static com.exadel.frs.ItemsBuilder.makeModel; +import static com.exadel.frs.ItemsBuilder.makeSubject; +import static com.exadel.frs.commonservice.enums.GlobalRole.USER; +import static java.time.LocalDateTime.now; +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.MILLIS; +import static java.util.UUID.randomUUID; import com.exadel.frs.commonservice.entity.Embedding; import com.exadel.frs.commonservice.entity.Img; import com.exadel.frs.commonservice.entity.Model; +import com.exadel.frs.commonservice.entity.ModelStatistic; +import com.exadel.frs.commonservice.entity.ResetPasswordToken; import com.exadel.frs.commonservice.entity.Subject; +import com.exadel.frs.commonservice.entity.User; +import com.exadel.frs.commonservice.enums.GlobalRole; import com.exadel.frs.commonservice.enums.ModelType; import com.exadel.frs.commonservice.repository.EmbeddingRepository; import com.exadel.frs.commonservice.repository.ImgRepository; import com.exadel.frs.commonservice.repository.ModelRepository; +import com.exadel.frs.commonservice.repository.ModelStatisticRepository; import com.exadel.frs.commonservice.repository.SubjectRepository; +import com.exadel.frs.commonservice.repository.UserRepository; import com.exadel.frs.repository.AppRepository; +import com.exadel.frs.repository.ResetPasswordTokenRepository; +import java.time.LocalDateTime; +import java.util.UUID; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; -import java.util.UUID; - -import static com.exadel.frs.ItemsBuilder.*; - @Service // TODO think about common helper for admin/core public class DbHelper { + @Value("${forgot-password.reset-password-token.expires}") + private long resetPasswordTokenExpires; + @Autowired AppRepository appRepository; @@ -36,13 +56,35 @@ public class DbHelper { @Autowired ImgRepository imgRepository; + @Autowired + UserRepository userRepository; + + @Autowired + ModelStatisticRepository modelStatisticRepository; + + @Autowired + ResetPasswordTokenRepository resetPasswordTokenRepository; + + @Autowired + PasswordEncoder encoder; + public Model insertModel() { - final String apiKey = UUID.randomUUID().toString(); + final String apiKey = randomUUID().toString(); var app = appRepository.save(makeApp(apiKey)); return modelRepository.save(makeModel(apiKey, ModelType.RECOGNITION, app)); } + public ModelStatistic insertModelStatistic(int requestCount, LocalDateTime createdDate, Model model) { + var statistic = ModelStatistic.builder() + .requestCount(requestCount) + .createdDate(createdDate) + .model(model) + .build(); + + return modelStatisticRepository.save(statistic); + } + public Subject insertSubject(Model model, String subjectName) { return insertSubject(model.getApiKey(), subjectName); } @@ -101,4 +143,50 @@ public Embedding insertEmbeddingWithImg(Subject subject, String calculator, doub public Img insertImg() { return imgRepository.save(makeImg()); } -} \ No newline at end of file + + public User insertUser(String email) { + return insertUser(email, USER); + } + + public User insertUser(String email, GlobalRole role) { + var user = createUser(email, role); + user.setEnabled(true); + return userRepository.saveAndFlush(user); + } + + public User insertUnconfirmedUser(String email) { + return insertUnconfirmedUser(email, USER); + } + + public User insertUnconfirmedUser(String email, GlobalRole role) { + var user = createUser(email, role); + user.setRegistrationToken(UUID.randomUUID().toString()); + return userRepository.saveAndFlush(user); + } + + private User createUser(String email, GlobalRole role) { + return User.builder() + .email(email) + .firstName("firstName") + .lastName("lastName") + .password(encoder.encode("1234567890")) + .guid(UUID.randomUUID().toString()) + .accountNonExpired(true) + .accountNonLocked(true) + .credentialsNonExpired(true) + .allowStatistics(true) + .globalRole(role) + .build(); + } + + public ResetPasswordToken insertResetPasswordToken(User user) { + var expiresIn = now(UTC).plus(resetPasswordTokenExpires, MILLIS); + var token = new ResetPasswordToken(expiresIn, user); + return resetPasswordTokenRepository.saveAndFlush(token); + } + + public ResetPasswordToken insertResetPasswordToken(User user, LocalDateTime expiresIn) { + var token = new ResetPasswordToken(expiresIn, user); + return resetPasswordTokenRepository.saveAndFlush(token); + } +} diff --git a/java/admin/src/test/java/com/exadel/frs/controller/AppControllerTest.java b/java/admin/src/test/java/com/exadel/frs/controller/AppControllerTest.java index 1285b1b14e..8aefbd0eef 100644 --- a/java/admin/src/test/java/com/exadel/frs/controller/AppControllerTest.java +++ b/java/admin/src/test/java/com/exadel/frs/controller/AppControllerTest.java @@ -18,6 +18,7 @@ import static com.exadel.frs.commonservice.enums.AppRole.OWNER; import static com.exadel.frs.commonservice.enums.AppRole.USER; +import static com.exadel.frs.system.global.Constants.ADMIN; import static com.exadel.frs.utils.TestUtils.USER_ID; import static com.exadel.frs.utils.TestUtils.buildExceptionResponse; import static com.exadel.frs.utils.TestUtils.buildUndefinedExceptionResponse; @@ -38,12 +39,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import com.exadel.frs.commonservice.exception.BasicException; -import com.exadel.frs.dto.ui.AppCreateDto; -import com.exadel.frs.dto.ui.AppResponseDto; -import com.exadel.frs.dto.ui.AppUpdateDto; -import com.exadel.frs.dto.ui.UserInviteDto; -import com.exadel.frs.dto.ui.UserRoleResponseDto; -import com.exadel.frs.dto.ui.UserRoleUpdateDto; +import com.exadel.frs.dto.AppCreateDto; +import com.exadel.frs.dto.AppResponseDto; +import com.exadel.frs.dto.AppUpdateDto; +import com.exadel.frs.dto.UserInviteDto; +import com.exadel.frs.dto.UserRoleResponseDto; +import com.exadel.frs.dto.UserRoleUpdateDto; import com.exadel.frs.commonservice.entity.App; import com.exadel.frs.commonservice.entity.UserAppRole; import com.exadel.frs.commonservice.enums.AppRole; @@ -56,8 +57,11 @@ import com.exadel.frs.system.security.config.WebSecurityConfig; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; +import lombok.SneakyThrows; import lombok.val; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -103,7 +107,7 @@ public void shouldReturnMessageAndCodeWhenAppNotFoundExceptionThrown() throws Ex when(appService.getApp(APP_GUID, USER_ID)).thenThrow(expectedException); String expectedContent = mapper.writeValueAsString(buildExceptionResponse(expectedException)); - mockMvc.perform(get( "/app/" + APP_GUID).with(user(buildUser()))) + mockMvc.perform(get(ADMIN + "/app/" + APP_GUID).with(user(buildUser()))) .andExpect(status().isNotFound()) .andExpect(content().string(expectedContent)); } @@ -115,14 +119,14 @@ public void shouldReturnMessageAndCodeWhenUnexpectedExceptionThrown() throws Exc when(appService.getApps(USER_ID)).thenThrow(expectedException); String expectedContent = mapper.writeValueAsString(buildUndefinedExceptionResponse(expectedException)); - mockMvc.perform(get("/apps").with(user(buildUser()))) + mockMvc.perform(get(ADMIN + "/apps").with(user(buildUser()))) .andExpect(status().isBadRequest()) .andExpect(content().string(expectedContent)); } @Test public void shouldReturnMessageAndCodeWhenAppNameIsMissing() throws Exception { - val request = post("/app") + val request = post(ADMIN + "/app") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -145,10 +149,10 @@ public void shouldReturn400AndErrorMessageWhenRenameAppToEmpty() throws Exceptio val expectedContent = "{\"message\":\"Application name cannot be empty\",\"code\":26}"; val bodyWithEmptyName = new AppUpdateDto(); - bodyWithEmptyName.setName(""); + bodyWithEmptyName.setName(null); val bodyWithNoName = new AppUpdateDto(); - val updateRequest = put("/app/" + APP_GUID) + val updateRequest = put(ADMIN + "/app/" + APP_GUID) .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -162,13 +166,14 @@ public void shouldReturn400AndErrorMessageWhenRenameAppToEmpty() throws Exceptio .andExpect(content().string(expectedContent)); } - @Test + @ParameterizedTest + @ValueSource(strings = {APP_NAME, "_[my_new app.]_"}) public void shouldReturnNewApp() throws Exception { val appCreateDto = AppCreateDto.builder() .name(APP_NAME) .build(); - val request = post("/app") + val request = post(ADMIN + "/app") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -195,7 +200,7 @@ public void shouldReturnUpdatedApp() throws Exception { .name(APP_NAME) .build(); - val request = put("/app/" + APP_GUID) + val request = put(ADMIN + "/app/" + APP_GUID) .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -218,7 +223,7 @@ public void shouldReturnUpdatedApp() throws Exception { @Test public void shouldReturnUpdatedWithApiKeyApp() throws Exception { - val request = put("/app/" + APP_GUID + "/apikey") + val request = put(ADMIN + "/app/" + APP_GUID + "/apikey") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -240,7 +245,7 @@ public void shouldReturnUpdatedWithApiKeyApp() throws Exception { @Test public void shouldReturnOkWhenDelete() throws Exception { - val request = delete("/app/" + APP_GUID) + val request = delete(ADMIN + "/app/" + APP_GUID) .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -253,7 +258,7 @@ public void shouldReturnOkWhenDelete() throws Exception { @Test public void shouldReturnGlobalRolesToAssign() throws Exception { - val request = get("/app/" + APP_GUID + "/assign-roles") + val request = get(ADMIN + "/app/" + APP_GUID + "/assign-roles") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -269,7 +274,7 @@ public void shouldReturnGlobalRolesToAssign() throws Exception { @Test public void shouldReturnAppUsers() throws Exception { - val request = get("/app/" + APP_GUID + "/roles") + val request = get(ADMIN + "/app/" + APP_GUID + "/roles") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -297,7 +302,7 @@ public void shouldReturnInvitedUser() throws Exception { .userEmail("email@test.com") .build(); - val request = post("/app/" + APP_GUID + "/invite") + val request = post(ADMIN + "/app/" + APP_GUID + "/invite") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -325,7 +330,7 @@ public void shouldReturnUpdatedUserAppRole() throws Exception { .userId(USER_GUID) .build(); - val request = put("/app/" + APP_GUID + "/role") + val request = put(ADMIN + "/app/" + APP_GUID + "/role") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -348,7 +353,7 @@ public void shouldReturnUpdatedUserAppRole() throws Exception { @Test public void shouldReturnOkWhenDeleteUserFromApp() throws Exception { - val request = delete("/app/" + APP_GUID + "/user/" + USER_GUID) + val request = delete(ADMIN + "/app/" + APP_GUID + "/user/" + USER_GUID) .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -358,4 +363,26 @@ public void shouldReturnOkWhenDeleteUserFromApp() throws Exception { mockMvc.perform(request) .andExpect(status().isOk()); } -} \ No newline at end of file + + @Test + @SneakyThrows + public void shouldReturn400WhenTryingToSaveAppThatContainsSpecialCharactersWithinName() { + var app = App.builder() + .id(APP_ID) + .name("\\new;app//") + .build(); + + val request = post(ADMIN + "/app") + .with(csrf()) + .with(user(buildUser())) + .contentType(MediaType.APPLICATION_JSON) + .content(mapper.writeValueAsString(app) + ); + + val expectedContent = "{\"message\":\"The name cannot contain the following special characters: ';', '/', '\\\\'\",\"code\":36}"; + + mockMvc.perform(request) + .andExpect(status().isBadRequest()) + .andExpect(content().string(expectedContent)); + } +} diff --git a/java/admin/src/test/java/com/exadel/frs/controller/ModelControllerTest.java b/java/admin/src/test/java/com/exadel/frs/controller/ModelControllerTest.java index 8ec9c9ecc9..d6074c00fc 100644 --- a/java/admin/src/test/java/com/exadel/frs/controller/ModelControllerTest.java +++ b/java/admin/src/test/java/com/exadel/frs/controller/ModelControllerTest.java @@ -16,10 +16,10 @@ package com.exadel.frs.controller; +import static com.exadel.frs.system.global.Constants.ADMIN; import static com.exadel.frs.utils.TestUtils.buildUser; import static java.util.UUID.randomUUID; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; @@ -33,11 +33,11 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -import com.exadel.frs.dto.ui.ModelCreateDto; -import com.exadel.frs.dto.ui.ModelResponseDto; -import com.exadel.frs.dto.ui.ModelUpdateDto; import com.exadel.frs.commonservice.entity.Model; +import com.exadel.frs.commonservice.repository.ModelStatisticRepository; +import com.exadel.frs.dto.ModelCreateDto; +import com.exadel.frs.dto.ModelResponseDto; +import com.exadel.frs.dto.ModelUpdateDto; import com.exadel.frs.mapper.MlModelMapper; import com.exadel.frs.service.ModelService; import com.exadel.frs.system.security.config.AuthServerConfig; @@ -45,8 +45,11 @@ import com.exadel.frs.system.security.config.WebSecurityConfig; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; +import lombok.SneakyThrows; import lombok.val; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -62,6 +65,9 @@ ) class ModelControllerTest { + @MockBean + private ModelStatisticRepository statisticRepository; + @MockBean private ModelService modelService; @@ -74,7 +80,6 @@ class ModelControllerTest { @Autowired private MockMvc mockMvc; - private static final String ORG_GUID = "org-guid"; private static final String APP_GUID = "app-guid"; private static final String MODEL_GUID = "model-guid"; private static final String MODEL_NAME = "model-name"; @@ -83,11 +88,11 @@ class ModelControllerTest { void shouldReturnMessageAndCodeWhenModelNameIsMissingOnUpdate() throws Exception { val expectedContent = "{\"message\":\"Model name cannot be empty\",\"code\":26}"; val bodyWithEmptyName = new ModelUpdateDto(); - bodyWithEmptyName.setName(""); + bodyWithEmptyName.setName(null); val bodyWithNoName = new ModelUpdateDto(); - val updateRequest = put( "/app/" + APP_GUID + "/model/" + MODEL_GUID) + val updateRequest = put(ADMIN + "/app/" + APP_GUID + "/model/" + MODEL_GUID) .with(csrf()) .with(user(buildUser())) .contentType(APPLICATION_JSON); @@ -103,44 +108,30 @@ void shouldReturnMessageAndCodeWhenModelNameIsMissingOnUpdate() throws Exception @Test void shouldReturnErrorMessageWhenNameIsMissingOnCreateNewModel() throws Exception { - val expectedContent = "{\"message\":\"Model name cannot be empty\",\"code\":26}"; - val bodyWithEmptyName = new ModelCreateDto(); - bodyWithEmptyName.setName(""); - bodyWithEmptyName.setType("RECOGNITION"); - val bodyWithNoName = new ModelCreateDto(); bodyWithNoName.setType("RECOGNITION"); - val createNewModelRequest = post("/app/" + APP_GUID + "/model") + val createNewModelRequest = post(ADMIN + "/app/" + APP_GUID + "/model") .with(csrf()) .with(user(buildUser())) .contentType(APPLICATION_JSON); - mockMvc.perform(createNewModelRequest.content(mapper.writeValueAsString(bodyWithEmptyName))) - .andExpect(status().isBadRequest()) - .andExpect(content().string(expectedContent)); - mockMvc.perform(createNewModelRequest.content(mapper.writeValueAsString(bodyWithNoName))) .andExpect(status().isBadRequest()) - .andExpect(content().string(expectedContent)); + .andExpect(content().string("{\"message\":\"Model name cannot be empty\",\"code\":26}")); } @Test void shouldReturnModel() throws Exception { - val request = get("/app/" + APP_GUID + "/model/" + MODEL_GUID) + val request = get(ADMIN + "/app/" + APP_GUID + "/model/" + MODEL_GUID) .with(csrf()) .with(user(buildUser())) .contentType(APPLICATION_JSON); - val model = Model.builder() - .name(MODEL_NAME) - .build(); - val responseDto = new ModelResponseDto(); responseDto.setName(MODEL_NAME); - when(modelService.getModel(eq(APP_GUID), eq(MODEL_GUID), anyLong())).thenReturn(model); - when(modelMapper.toResponseDto(any(Model.class), eq(APP_GUID))).thenReturn(responseDto); + when(modelService.getModelDto(eq(APP_GUID), eq(MODEL_GUID), anyLong())).thenReturn(responseDto); mockMvc.perform(request) .andExpect(status().isOk()) @@ -149,33 +140,29 @@ void shouldReturnModel() throws Exception { @Test void shouldReturnModels() throws Exception { - val request = get("/app/" + APP_GUID + "/models") + val request = get(ADMIN + "/app/" + APP_GUID + "/models") .with(csrf()) .with(user(buildUser())) .contentType(APPLICATION_JSON); - val model = Model.builder() - .name(MODEL_NAME) - .build(); - val responseDto = new ModelResponseDto(); responseDto.setName(MODEL_NAME); - when(modelService.getModels(eq(APP_GUID), anyLong())).thenReturn(List.of(model, model)); - when(modelMapper.toResponseDto(anyList(), eq(APP_GUID))).thenReturn(List.of(responseDto, responseDto)); + when(modelService.getModels(eq(APP_GUID), anyLong())).thenReturn(List.of(responseDto, responseDto)); mockMvc.perform(request) .andExpect(status().isOk()) .andExpect(content().string(mapper.writeValueAsString(List.of(responseDto, responseDto)))); } - @Test + @ParameterizedTest + @ValueSource(strings = {MODEL_NAME, "_model_-.[]"}) void shouldReturnCreatedModel() throws Exception { val createDto = new ModelCreateDto(); createDto.setName(MODEL_NAME); createDto.setType("RECOGNITION"); - val createRequest = post("/app/" + APP_GUID + "/model") + val createRequest = post(ADMIN + "/app/" + APP_GUID + "/model") .with(csrf()) .with(user(buildUser())) .contentType(APPLICATION_JSON) @@ -201,7 +188,7 @@ void shouldReturnUpdatedModel() throws Exception { val updateDto = new ModelUpdateDto(); updateDto.setName(MODEL_NAME); - val createRequest = put("/app/" + APP_GUID + "/model/" + MODEL_GUID) + val createRequest = put(ADMIN + "/app/" + APP_GUID + "/model/" + MODEL_GUID) .with(csrf()) .with(user(buildUser())) .contentType(APPLICATION_JSON) @@ -227,22 +214,17 @@ void shouldReturnUpdatedWithApiKeyModel() throws Exception { val updateDto = new ModelUpdateDto(); updateDto.setName(MODEL_NAME); - val request = put("/app/" + APP_GUID + "/model/" + MODEL_GUID + "/apikey") + val request = put(ADMIN + "/app/" + APP_GUID + "/model/" + MODEL_GUID + "/apikey") .with(csrf()) .with(user(buildUser())) .contentType(APPLICATION_JSON); val newApiKey = randomUUID().toString(); - val model = Model.builder() - .apiKey(newApiKey) - .build(); - val responseDto = new ModelResponseDto(); responseDto.setApiKey(newApiKey); - when(modelService.getModel(eq(APP_GUID), eq(MODEL_GUID), anyLong())).thenReturn(model); - when(modelMapper.toResponseDto(any(Model.class), eq(APP_GUID))).thenReturn(responseDto); + when(modelService.getModelDto(eq(APP_GUID), eq(MODEL_GUID), anyLong())).thenReturn(responseDto); mockMvc.perform(request) .andExpect(status().isOk()) @@ -254,7 +236,7 @@ void shouldReturnOkWhenDeleteModel() throws Exception { val updateDto = new ModelUpdateDto(); updateDto.setName(MODEL_NAME); - val request = delete("/app/" + APP_GUID + "/model/" + MODEL_GUID) + val request = delete(ADMIN + "/app/" + APP_GUID + "/model/" + MODEL_GUID) .with(csrf()) .with(user(buildUser())) .contentType(APPLICATION_JSON); @@ -264,4 +246,20 @@ void shouldReturnOkWhenDeleteModel() throws Exception { mockMvc.perform(request) .andExpect(status().isOk()); } -} \ No newline at end of file + + @Test + @SneakyThrows + void shouldReturnErrorMessageWhenNameContainsSpecialCharactersOnCreateNewModel() { + val bodyWithEmptyName = new ModelCreateDto(); + bodyWithEmptyName.setName("\\new;model//"); + bodyWithEmptyName.setType("RECOGNITION"); + + mockMvc.perform(post(ADMIN + "/app/" + APP_GUID + "/model") + .with(csrf()) + .with(user(buildUser())) + .contentType(APPLICATION_JSON).content(mapper.writeValueAsString(bodyWithEmptyName))) + .andExpect(status().isBadRequest()) + .andExpect(content().string("{\"message\":\"The name cannot contain the following special characters: ';', '/', '\\\\'\"," + + "\"code\":36}")); + } +} diff --git a/java/admin/src/test/java/com/exadel/frs/controller/UserControllerTest.java b/java/admin/src/test/java/com/exadel/frs/controller/UserControllerTest.java index eba8b9ec59..881372e8a0 100644 --- a/java/admin/src/test/java/com/exadel/frs/controller/UserControllerTest.java +++ b/java/admin/src/test/java/com/exadel/frs/controller/UserControllerTest.java @@ -19,6 +19,7 @@ import static com.exadel.frs.commonservice.handler.CommonExceptionCode.EMPTY_REQUIRED_FIELD; import static com.exadel.frs.commonservice.handler.CrudExceptionCode.INCORRECT_USER_PASSWORD; import static com.exadel.frs.commonservice.handler.CrudExceptionCode.VALIDATION_CONSTRAINT_VIOLATION; +import static com.exadel.frs.system.global.Constants.ADMIN; import static com.exadel.frs.utils.TestUtils.buildUser; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.is; @@ -44,11 +45,11 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import com.exadel.frs.commonservice.exception.EmptyRequiredFieldException; -import com.exadel.frs.dto.ui.ChangePasswordDto; -import com.exadel.frs.dto.ui.UserAutocompleteDto; -import com.exadel.frs.dto.ui.UserCreateDto; -import com.exadel.frs.dto.ui.UserResponseDto; -import com.exadel.frs.dto.ui.UserUpdateDto; +import com.exadel.frs.dto.ChangePasswordDto; +import com.exadel.frs.dto.UserAutocompleteDto; +import com.exadel.frs.dto.UserCreateDto; +import com.exadel.frs.dto.UserResponseDto; +import com.exadel.frs.dto.UserUpdateDto; import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.exception.AccessDeniedException; import com.exadel.frs.exception.IncorrectUserPasswordException; @@ -57,6 +58,7 @@ import com.exadel.frs.mapper.UserMapper; import com.exadel.frs.service.AppService; import com.exadel.frs.service.ModelService; +import com.exadel.frs.service.ResetPasswordTokenService; import com.exadel.frs.service.UserService; import com.exadel.frs.system.security.config.AuthServerConfig; import com.exadel.frs.system.security.config.ResourceServerConfig; @@ -105,16 +107,19 @@ public class UserControllerTest { @MockBean private UserGlobalRoleMapper userGlobalRoleMapper; + @MockBean + private ResetPasswordTokenService resetPasswordTokenService; + @Autowired private MockMvc mockMvc; @Test void shouldReturnErrorMessageWhenUpdateFirstNameIsEmpty() throws Exception { - val expectedContent = "{\"message\":\"" + String.format(EmptyRequiredFieldException.MESSAGE, "firstName") + "\",\"code\":5}"; + val expectedContent = "{\"message\":\"" + String.format(EmptyRequiredFieldException.MESSAGE, "firstName") + "\",\"code\":26}"; val bodyWithEmptyFirstName = new UserUpdateDto(); bodyWithEmptyFirstName.setLastName("gdsag"); - val createNewModelRequest = put("/user/update") + val createNewModelRequest = put(ADMIN + "/user/update") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -126,11 +131,11 @@ void shouldReturnErrorMessageWhenUpdateFirstNameIsEmpty() throws Exception { @Test void shouldReturnErrorMessageWhenUpdateLastNameIsEmpty() throws Exception { - val expectedContent = "{\"message\":\"" + String.format(EmptyRequiredFieldException.MESSAGE, "lastName") + "\",\"code\":5}"; + val expectedContent = "{\"message\":\"" + String.format(EmptyRequiredFieldException.MESSAGE, "lastName") + "\",\"code\":26}"; val bodyWithEmptyLastName = new UserUpdateDto(); bodyWithEmptyLastName.setFirstName("gdsag"); - val createNewModelRequest = put("/user/update") + val createNewModelRequest = put(ADMIN + "/user/update") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -156,7 +161,7 @@ static Stream verifyChangePasswordValidationExceptionsProvider() { void testChangePasswordValidationExceptions(String oldPwd, String newPwd) throws Exception { // given ChangePasswordDto bodyWithEmptyPassword = new ChangePasswordDto(oldPwd, newPwd); - val createNewModelRequest = put("/user/me/password") + val createNewModelRequest = put(ADMIN + "/user/me/password") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -178,7 +183,7 @@ void testChangePasswordIncorrectPassword() throws Exception { String oldPwd = "oldPassword"; String newPwd = "newPassword"; ChangePasswordDto bodyWithIncorrectPassword = new ChangePasswordDto(oldPwd, newPwd); - val createNewModelRequest = put("/user/me/password") + val createNewModelRequest = put(ADMIN + "/user/me/password") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -197,7 +202,7 @@ void testChangePassword() throws Exception { String oldPwd = "oldPassword"; String newPwd = "newPassword"; ChangePasswordDto bodyWithIncorrectPassword = new ChangePasswordDto(oldPwd, newPwd); - val createNewModelRequest = put("/user/me/password") + val createNewModelRequest = put(ADMIN + "/user/me/password") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -215,7 +220,7 @@ void shouldReturnUpdatedUser() throws Exception { updateDto.setLastName("gdsag"); updateDto.setFirstName("test"); - val createRequest = put("/user/update") + val createRequest = put(ADMIN + "/user/update") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -239,7 +244,7 @@ void shouldReturnOkWhenDeleteUser() throws Exception { updateDto.setLastName("gdsag"); updateDto.setFirstName("test"); - val deleteRequest = delete("/user/" + USER_GUID) + val deleteRequest = delete(ADMIN + "/user/" + USER_GUID) .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -256,7 +261,7 @@ void shouldReturnOkWhenDeleteUser() throws Exception { @Test void shouldReturnAutocomplete() throws Exception { - val createRequest = get("/user/autocomplete") + val createRequest = get(ADMIN + "/user/autocomplete") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -282,7 +287,7 @@ void shouldReturnAutocomplete() throws Exception { @Test void shouldReturnSendRedirect() throws Exception { - val createRequest = get("/user/registration/confirm") + val createRequest = get(ADMIN + "/user/registration/confirm") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) @@ -297,7 +302,7 @@ void shouldReturnSendRedirect() throws Exception { void shouldReturnErrorMessageWhenNoUser() throws Exception { val expectedContent = "{\"message\":\"" + AccessDeniedException.MESSAGE + "\",\"code\":1}"; - val createNewModelRequest = get("/user/me") + val createNewModelRequest = get(ADMIN + "/user/me") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON); @@ -313,7 +318,7 @@ void shouldReturnErrorMessageWhenNoUser() throws Exception { void shouldReturnOkWhenUserNotEnabled() throws Exception { val createDto = new UserCreateDto("email", "name", "last", "password", false); - val createRequest = post("/user/register") + val createRequest = post(ADMIN + "/user/register") .with(csrf()) .with(user(buildUser())) .contentType(MediaType.APPLICATION_JSON) diff --git a/java/admin/src/test/java/com/exadel/frs/security/OAuthMvcTest.java b/java/admin/src/test/java/com/exadel/frs/security/OAuthMvcTest.java index 8372baaeb2..0b7de6f79e 100644 --- a/java/admin/src/test/java/com/exadel/frs/security/OAuthMvcTest.java +++ b/java/admin/src/test/java/com/exadel/frs/security/OAuthMvcTest.java @@ -16,6 +16,7 @@ package com.exadel.frs.security; +import static com.exadel.frs.system.global.Constants.ADMIN; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.mockito.Mockito.when; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; @@ -104,7 +105,7 @@ private Cookie getCookie(String username, String password) throws Exception { params.add("password", password); ResultActions result - = mockMvc.perform(post("/oauth/token") + = mockMvc.perform(post(ADMIN + "/oauth/token") .params(params) .with(httpBasic("CommonClientId", "password")) .accept("application/json;charset=UTF-8")) @@ -116,23 +117,23 @@ private Cookie getCookie(String username, String password) throws Exception { @Test void availableOnlyWithCookie() throws Exception { - mockMvc.perform(get("/user/me")) + mockMvc.perform(get(ADMIN + "/user/me")) .andExpect(status().isUnauthorized()); var cookie = getCookie(userEmail, "test1"); - mockMvc.perform(get("/user/me") + mockMvc.perform(get(ADMIN + "/user/me") .cookie(cookie)) .andExpect(status().isOk()); } @Test void ignoresCaseWhenLogin() throws Exception { - mockMvc.perform(get("/user/me")) + mockMvc.perform(get(ADMIN + "/user/me")) .andExpect(status().isUnauthorized()); val cookie = getCookie(userEmail.toUpperCase(), "test1"); - mockMvc.perform(get("/user/me") + mockMvc.perform(get(ADMIN + "/user/me") .cookie(cookie)) .andExpect(status().isOk()); } @@ -146,7 +147,7 @@ private void createUser(String email) throws Exception { " \"password\": \"test1\"\n" + "}"; - mockMvc.perform(post("/user/register") + mockMvc.perform(post(ADMIN + "/user/register") .contentType("application/json") .content(employeeString) .accept("application/json")) diff --git a/java/admin/src/test/java/com/exadel/frs/AppServiceTest.java b/java/admin/src/test/java/com/exadel/frs/service/AppServiceTest.java similarity index 95% rename from java/admin/src/test/java/com/exadel/frs/AppServiceTest.java rename to java/admin/src/test/java/com/exadel/frs/service/AppServiceTest.java index d3b5c40a08..88e2d6b775 100644 --- a/java/admin/src/test/java/com/exadel/frs/AppServiceTest.java +++ b/java/admin/src/test/java/com/exadel/frs/service/AppServiceTest.java @@ -14,18 +14,17 @@ * permissions and limitations under the License. */ -package com.exadel.frs; +package com.exadel.frs.service; import com.exadel.frs.commonservice.entity.*; import com.exadel.frs.commonservice.enums.AppRole; import com.exadel.frs.commonservice.enums.GlobalRole; -import com.exadel.frs.dto.ui.AppCreateDto; -import com.exadel.frs.dto.ui.AppUpdateDto; -import com.exadel.frs.dto.ui.UserInviteDto; -import com.exadel.frs.dto.ui.UserRoleUpdateDto; +import com.exadel.frs.dto.AppCreateDto; +import com.exadel.frs.dto.AppUpdateDto; +import com.exadel.frs.dto.UserInviteDto; +import com.exadel.frs.dto.UserRoleUpdateDto; import com.exadel.frs.exception.InsufficientPrivilegesException; import com.exadel.frs.exception.NameIsNotUniqueException; -import com.exadel.frs.exception.SelfRoleChangeException; import com.exadel.frs.exception.UserAlreadyHasAccessToAppException; import com.exadel.frs.repository.AppRepository; import com.exadel.frs.service.AppService; @@ -116,7 +115,7 @@ void successGetAppsForGlobalAdmin() { .guid(APPLICATION_GUID) .build(); - when(appRepositoryMock.findAll()).thenReturn(List.of(app)); + when(appRepositoryMock.findAllByOrderByNameAsc()).thenReturn(List.of(app)); when(userServiceMock.getUser(USER_ID)).thenReturn(user); val result = appService.getApps(USER_ID); @@ -239,34 +238,6 @@ void failUpdateUserAppSelfRoleOwnerChange() { verifyNoMoreInteractions(appRepositoryMock); } - @Test - void failUpdateAppSelfRoleChange() { - val userRoleUpdateDto = UserRoleUpdateDto.builder() - .userId("userGuid") - .role(AppRole.USER.toString()) - .build(); - val user = user(USER_ID, USER); - - val app = App.builder() - .name("name") - .guid(APPLICATION_GUID) - .build(); - - when(appRepositoryMock.findByGuid(APPLICATION_GUID)).thenReturn(Optional.of(app)); - when(userServiceMock.getUserByGuid(any())).thenReturn(user); - when(userServiceMock.getUser(USER_ID)).thenReturn(user); - - assertThatThrownBy(() -> appService.updateUserAppRole( - userRoleUpdateDto, - APPLICATION_GUID, - USER_ID - )).isInstanceOf(SelfRoleChangeException.class); - - verify(authManagerMock).verifyWritePrivilegesToApp(user, app, true); - verify(authManagerMock).verifyReadPrivilegesToApp(user, app); - verifyNoMoreInteractions(authManagerMock); - } - @Test void failUpdateAppNameIsNotUnique() { val appUpdateDto = AppUpdateDto.builder() diff --git a/java/admin/src/test/java/com/exadel/frs/ModelServiceTest.java b/java/admin/src/test/java/com/exadel/frs/service/ModelServiceTest.java similarity index 70% rename from java/admin/src/test/java/com/exadel/frs/ModelServiceTest.java rename to java/admin/src/test/java/com/exadel/frs/service/ModelServiceTest.java index 1533e07602..acd2597bff 100644 --- a/java/admin/src/test/java/com/exadel/frs/ModelServiceTest.java +++ b/java/admin/src/test/java/com/exadel/frs/service/ModelServiceTest.java @@ -14,46 +14,45 @@ * permissions and limitations under the License. */ -package com.exadel.frs; +package com.exadel.frs.service; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.emptyOrNullString; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import com.exadel.frs.commonservice.repository.SubjectRepository; -import com.exadel.frs.dto.ui.ModelCloneDto; -import com.exadel.frs.dto.ui.ModelCreateDto; -import com.exadel.frs.dto.ui.ModelUpdateDto; import com.exadel.frs.commonservice.entity.App; import com.exadel.frs.commonservice.entity.Model; +import com.exadel.frs.commonservice.projection.ModelProjection; import com.exadel.frs.commonservice.entity.User; -import com.exadel.frs.commonservice.enums.AppModelAccess; -import com.exadel.frs.exception.NameIsNotUniqueException; +import com.exadel.frs.commonservice.repository.ImgRepository; import com.exadel.frs.commonservice.repository.ModelRepository; +import com.exadel.frs.commonservice.repository.ModelStatisticRepository; +import com.exadel.frs.commonservice.repository.SubjectRepository; +import com.exadel.frs.dto.ModelCloneDto; +import com.exadel.frs.dto.ModelCreateDto; +import com.exadel.frs.dto.ModelResponseDto; +import com.exadel.frs.dto.ModelUpdateDto; +import com.exadel.frs.exception.NameIsNotUniqueException; +import com.exadel.frs.mapper.MlModelMapper; import com.exadel.frs.service.AppService; import com.exadel.frs.service.ModelCloneService; import com.exadel.frs.service.ModelService; import com.exadel.frs.service.UserService; import com.exadel.frs.system.security.AuthorizationManager; +import lombok.val; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.jdbc.core.JdbcTemplate; import java.util.List; import java.util.Optional; import java.util.Random; -import lombok.val; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.springframework.jdbc.core.JdbcTemplate; +import static com.exadel.frs.commonservice.enums.ModelType.RECOGNITION; +import static java.time.LocalDateTime.now; +import static java.util.UUID.randomUUID; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.*; class ModelServiceTest { @@ -71,7 +70,10 @@ class ModelServiceTest { private ModelService modelService; private UserService userServiceMock; private ModelCloneService modelCloneService; + private MlModelMapper modelMapper; + private ImgRepository imgRepository; private final SubjectRepository subjectRepositry; + private final ModelStatisticRepository statisticRepository; private final JdbcTemplate jdbcTemplate; private AuthorizationManager authManager; @@ -84,6 +86,9 @@ class ModelServiceTest { subjectRepositry = mock(SubjectRepository.class); jdbcTemplate = mock(JdbcTemplate.class); modelCloneService = mock(ModelCloneService.class); + modelMapper = mock(MlModelMapper.class); + imgRepository = mock(ImgRepository.class); + statisticRepository = mock(ModelStatisticRepository.class); modelService = new ModelService( modelRepositoryMock, @@ -92,33 +97,40 @@ class ModelServiceTest { userServiceMock, subjectRepositry, jdbcTemplate, - modelCloneService + modelCloneService, + modelMapper, + imgRepository, + statisticRepository ); } @Test void successGetModel() { val app = App.builder() - .id(APPLICATION_ID) - .build(); + .id(APPLICATION_ID) + .build(); val model = Model.builder() - .id(MODEL_ID) - .guid(MODEL_GUID) - .app(app) - .build(); + .id(MODEL_ID) + .guid(MODEL_GUID) + .app(app) + .build(); val user = User.builder() - .id(USER_ID) - .build(); + .id(USER_ID) + .build(); + + val modelDto = ModelResponseDto.builder() + .id(MODEL_GUID) + .build(); when(modelRepositoryMock.findByGuid(MODEL_GUID)).thenReturn(Optional.of(model)); when(userServiceMock.getUser(USER_ID)).thenReturn(user); + when(modelMapper.toResponseDto(model, APPLICATION_GUID)).thenReturn(modelDto); - val result = modelService.getModel(APPLICATION_GUID, MODEL_GUID, USER_ID); + val result = modelService.getModelDto(APPLICATION_GUID, MODEL_GUID, USER_ID); - assertThat(result.getGuid(), is(MODEL_GUID)); - assertThat(result.getId(), is(MODEL_ID)); + assertThat(result.getId(), is(MODEL_GUID)); verify(authManager).verifyReadPrivilegesToApp(user, app); verify(authManager).verifyAppHasTheModel(APPLICATION_GUID, model); @@ -128,22 +140,23 @@ void successGetModel() { @Test void successGetModels() { val app = App.builder() - .id(APPLICATION_ID) - .build(); + .id(APPLICATION_ID) + .build(); - val model = Model.builder() - .id(MODEL_ID) - .guid(MODEL_GUID) - .app(app) - .build(); + val model = new ModelProjection(MODEL_GUID, "1REC", MODEL_API_KEY, RECOGNITION, now()); val user = User.builder() - .id(USER_ID) - .build(); + .id(USER_ID) + .build(); + + val modelDto = ModelResponseDto.builder() + .id(MODEL_GUID) + .build(); when(modelRepositoryMock.findAllByAppId(anyLong())).thenReturn(List.of(model)); when(appServiceMock.getApp(APPLICATION_GUID)).thenReturn(app); when(userServiceMock.getUser(USER_ID)).thenReturn(user); + when(modelMapper.toResponseDto(model, MODEL_API_KEY)).thenReturn(modelDto); val result = modelService.getModels(APPLICATION_GUID, USER_ID); @@ -156,18 +169,18 @@ void successGetModels() { @Test void successCreateModel() { val modelCreateDto = ModelCreateDto.builder() - .name("model-name") - .type("RECOGNITION") - .build(); + .name("model-name") + .type("RECOGNITION") + .build(); val app = App.builder() - .id(APPLICATION_ID) - .guid(APPLICATION_GUID) - .build(); + .id(APPLICATION_ID) + .guid(APPLICATION_GUID) + .build(); val user = User.builder() - .id(USER_ID) - .build(); + .id(USER_ID) + .build(); when(appServiceMock.getApp(APPLICATION_GUID)).thenReturn(app); when(userServiceMock.getUser(USER_ID)).thenReturn(user); @@ -175,7 +188,7 @@ void successCreateModel() { modelService.createRecognitionModel(modelCreateDto, APPLICATION_GUID, USER_ID); val varArgs = ArgumentCaptor.forClass(Model.class); - verify(modelRepositoryMock).existsByNameAndAppId("model-name", APPLICATION_ID); + verify(modelRepositoryMock).existsByUniqueNameAndAppId("model-name", APPLICATION_ID); verify(modelRepositoryMock).save(varArgs.capture()); verify(authManager).verifyWritePrivilegesToApp(user, app); verifyNoMoreInteractions(modelRepositoryMock, authManager); @@ -187,16 +200,16 @@ void successCreateModel() { @Test void failCreateModelNameIsNotUnique() { val modelCreateDto = ModelCreateDto.builder() - .name("model-name") - .build(); + .name("model-name") + .build(); val app = App.builder() - .id(APPLICATION_ID) - .guid(APPLICATION_GUID) - .build(); + .id(APPLICATION_ID) + .guid(APPLICATION_GUID) + .build(); when(appServiceMock.getApp(anyString())).thenReturn(app); - when(modelRepositoryMock.existsByNameAndAppId(anyString(), anyLong())).thenReturn(true); + when(modelRepositoryMock.existsByUniqueNameAndAppId(anyString(), anyLong())).thenReturn(true); assertThatThrownBy(() -> modelService.createRecognitionModel(modelCreateDto, APPLICATION_GUID, USER_ID) @@ -206,8 +219,8 @@ void failCreateModelNameIsNotUnique() { @Test void successCloneModel() { val user = User.builder() - .id(USER_ID) - .build(); + .id(USER_ID) + .build(); val modelCloneDto = ModelCloneDto.builder() .name("name_of_clone") @@ -224,7 +237,6 @@ void successCloneModel() { .guid(MODEL_GUID) .app(app) .build(); - repoModel.addAppModelAccess(app, AppModelAccess.READONLY); val cloneModel = Model.builder() .id(new Random().nextLong()) @@ -242,7 +254,7 @@ void successCloneModel() { val clonedModel = modelService.cloneModel(modelCloneDto, APPLICATION_GUID, MODEL_GUID, USER_ID); verify(modelRepositoryMock).findByGuid(MODEL_GUID); - verify(modelRepositoryMock).existsByNameAndAppId("name_of_clone", APPLICATION_ID); + verify(modelRepositoryMock).existsByUniqueNameAndAppId("name_of_clone", APPLICATION_ID); verify(modelCloneService).cloneModel(any(Model.class), any(ModelCloneDto.class)); verify(authManager).verifyAppHasTheModel(APPLICATION_GUID, repoModel); verify(authManager).verifyWritePrivilegesToApp(user, app); @@ -271,7 +283,7 @@ void failCloneModelNameIsNotUnique() { when(modelRepositoryMock.findByGuid(anyString())).thenReturn(Optional.of(repoModel)); when(appServiceMock.getApp(anyString())).thenReturn(app); - when(modelRepositoryMock.existsByNameAndAppId(anyString(), anyLong())).thenReturn(true); + when(modelRepositoryMock.existsByUniqueNameAndAppId(anyString(), anyLong())).thenReturn(true); assertThatThrownBy(() -> modelService.cloneModel(modelCloneDto, APPLICATION_GUID, MODEL_GUID, USER_ID) @@ -281,26 +293,24 @@ void failCloneModelNameIsNotUnique() { @Test void successUpdateModel() { ModelUpdateDto modelUpdateDto = ModelUpdateDto.builder() - .name("new_name") - .build(); + .name("new_name") + .build(); val app = App.builder() - .id(APPLICATION_ID) - .guid(APPLICATION_GUID) - .build(); + .id(APPLICATION_ID) + .guid(APPLICATION_GUID) + .build(); val repoModel = Model.builder() - .id(MODEL_ID) - .name("name") - .guid(MODEL_GUID) - .app(app) - .build(); + .id(MODEL_ID) + .name("name") + .guid(MODEL_GUID) + .app(app) + .build(); val user = User.builder() - .id(USER_ID) - .build(); - - repoModel.addAppModelAccess(app, AppModelAccess.READONLY); + .id(USER_ID) + .build(); when(modelRepositoryMock.findByGuid(MODEL_GUID)).thenReturn(Optional.of(repoModel)); when(appServiceMock.getApp(APPLICATION_GUID)).thenReturn(app); @@ -309,7 +319,7 @@ void successUpdateModel() { modelService.updateModel(modelUpdateDto, APPLICATION_GUID, MODEL_GUID, USER_ID); verify(modelRepositoryMock).findByGuid(MODEL_GUID); - verify(modelRepositoryMock).existsByNameAndAppId("new_name", APPLICATION_ID); + verify(modelRepositoryMock).existsByUniqueNameAndAppId("new_name", APPLICATION_ID); verify(modelRepositoryMock).save(any(Model.class)); verify(authManager).verifyReadPrivilegesToApp(user, app); verify(authManager).verifyAppHasTheModel(APPLICATION_GUID, repoModel); @@ -322,24 +332,24 @@ void successUpdateModel() { @Test void failUpdateModelNameIsNotUnique() { val modelUpdateDto = ModelUpdateDto.builder() - .name("new_name") - .build(); + .name("new_name") + .build(); val app = App.builder() - .id(APPLICATION_ID) - .guid(APPLICATION_GUID) - .build(); + .id(APPLICATION_ID) + .guid(APPLICATION_GUID) + .build(); val repoModel = Model.builder() - .id(MODEL_ID) - .name("name") - .guid(MODEL_GUID) - .app(app) - .build(); + .id(MODEL_ID) + .name("name") + .guid(MODEL_GUID) + .app(app) + .build(); when(modelRepositoryMock.findByGuid(anyString())).thenReturn(Optional.of(repoModel)); when(appServiceMock.getApp(anyString())).thenReturn(app); - when(modelRepositoryMock.existsByNameAndAppId(anyString(), anyLong())).thenReturn(true); + when(modelRepositoryMock.existsByUniqueNameAndAppId(anyString(), anyLong())).thenReturn(true); assertThatThrownBy(() -> modelService.updateModel(modelUpdateDto, APPLICATION_GUID, MODEL_GUID, USER_ID) @@ -349,21 +359,21 @@ void failUpdateModelNameIsNotUnique() { @Test void successRegenerateApiKey() { val app = App.builder() - .id(APPLICATION_ID) - .guid(APPLICATION_GUID) - .apiKey(APPLICATION_API_KEY) - .build(); + .id(APPLICATION_ID) + .guid(APPLICATION_GUID) + .apiKey(APPLICATION_API_KEY) + .build(); val model = Model.builder() - .id(MODEL_ID) - .guid(MODEL_GUID) - .apiKey(MODEL_API_KEY) - .app(app) - .build(); + .id(MODEL_ID) + .guid(MODEL_GUID) + .apiKey(MODEL_API_KEY) + .app(app) + .build(); val user = User.builder() - .id(USER_ID) - .build(); + .id(USER_ID) + .build(); when(modelRepositoryMock.findByGuid(MODEL_GUID)).thenReturn(Optional.of(model)); when(userServiceMock.getUser(USER_ID)).thenReturn(user); @@ -384,21 +394,21 @@ void successDeleteModel() { val modelKey = "model_key"; val app = App.builder() - .id(APPLICATION_ID) - .guid(APPLICATION_GUID) - .apiKey(appKey) - .build(); + .id(APPLICATION_ID) + .guid(APPLICATION_GUID) + .apiKey(appKey) + .build(); val model = Model.builder() - .id(MODEL_ID) - .guid(MODEL_GUID) - .apiKey(modelKey) - .app(app) - .build(); + .id(MODEL_ID) + .guid(MODEL_GUID) + .apiKey(modelKey) + .app(app) + .build(); val user = User.builder() - .id(USER_ID) - .build(); + .id(USER_ID) + .build(); when(modelRepositoryMock.findByGuid(MODEL_GUID)).thenReturn(Optional.of(model)); when(userServiceMock.getUser(USER_ID)).thenReturn(user); diff --git a/java/admin/src/test/java/com/exadel/frs/ModelServiceTestIT.java b/java/admin/src/test/java/com/exadel/frs/service/ModelServiceTestIT.java similarity index 68% rename from java/admin/src/test/java/com/exadel/frs/ModelServiceTestIT.java rename to java/admin/src/test/java/com/exadel/frs/service/ModelServiceTestIT.java index 3b7546a50e..15d1bf093f 100644 --- a/java/admin/src/test/java/com/exadel/frs/ModelServiceTestIT.java +++ b/java/admin/src/test/java/com/exadel/frs/service/ModelServiceTestIT.java @@ -14,15 +14,16 @@ * permissions and limitations under the License. */ -package com.exadel.frs; +package com.exadel.frs.service; -import com.exadel.frs.commonservice.entity.EmbeddingProjection; +import com.exadel.frs.DbHelper; +import com.exadel.frs.EmbeddedPostgreSQLTest; +import com.exadel.frs.commonservice.projection.EmbeddingProjection; import com.exadel.frs.commonservice.entity.Model; import com.exadel.frs.commonservice.entity.Subject; import com.exadel.frs.commonservice.repository.EmbeddingRepository; import com.exadel.frs.commonservice.repository.ImgRepository; import com.exadel.frs.commonservice.repository.SubjectRepository; -import com.exadel.frs.service.ModelService; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -32,7 +33,10 @@ import java.util.List; import java.util.UUID; +import org.springframework.transaction.annotation.Transactional; +import static java.time.LocalDateTime.now; +import static java.time.ZoneOffset.UTC; import static org.assertj.core.api.Assertions.assertThat; @ExtendWith(SpringExtension.class) @@ -54,6 +58,30 @@ class ModelServiceTestIT extends EmbeddedPostgreSQLTest { @Autowired ModelService modelService; + @Test + @Transactional + void testGetSummarizedByDayModelStatistics() { + var user = dbHelper.insertUser("john@gmail.com"); + var model = dbHelper.insertModel(); + var app = model.getApp(); + + var statisticsBefore = modelService.getSummarizedByDayModelStatistics(app.getGuid(), model.getGuid(), user.getId()); + + dbHelper.insertModelStatistic(3, now(UTC).minusMonths(1), model); + dbHelper.insertModelStatistic(5, now(UTC).minusMonths(1), model); + dbHelper.insertModelStatistic(8, now(UTC).minusMonths(7), model); + dbHelper.insertModelStatistic(4, now(UTC).minusMonths(4), model); + dbHelper.insertModelStatistic(9, now(UTC).plusMonths(1), model); + + var statisticsAfter = modelService.getSummarizedByDayModelStatistics(app.getGuid(), model.getGuid(), user.getId()); + + assertThat(statisticsBefore).isEmpty(); + assertThat(statisticsAfter).hasSize(2); + + assertThat(statisticsAfter.get(0).requestCount()).isEqualTo(8); + assertThat(statisticsAfter.get(1).requestCount()).isEqualTo(4); + } + @Test void testCloneSubjects() { final Model model = dbHelper.insertModel(); @@ -86,8 +114,8 @@ private void compareEmbeddings(List originals, List originals, List clones) { diff --git a/java/admin/src/test/java/com/exadel/frs/service/ResetPasswordTokenServiceTestIT.java b/java/admin/src/test/java/com/exadel/frs/service/ResetPasswordTokenServiceTestIT.java new file mode 100644 index 0000000000..7dc4f8fe18 --- /dev/null +++ b/java/admin/src/test/java/com/exadel/frs/service/ResetPasswordTokenServiceTestIT.java @@ -0,0 +1,256 @@ +package com.exadel.frs.service; + +import static java.time.LocalDateTime.now; +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.MILLIS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.exadel.frs.DbHelper; +import com.exadel.frs.EmbeddedPostgreSQLTest; +import com.exadel.frs.commonservice.repository.UserRepository; +import com.exadel.frs.exception.InvalidResetPasswordTokenException; +import com.exadel.frs.exception.UserDoesNotExistException; +import com.exadel.frs.repository.ResetPasswordTokenRepository; +import com.exadel.frs.service.ResetPasswordTokenService; +import com.icegreen.greenmail.configuration.GreenMailConfiguration; +import com.icegreen.greenmail.junit5.GreenMailExtension; +import com.icegreen.greenmail.util.GreenMailUtil; +import com.icegreen.greenmail.util.ServerSetupTest; +import java.util.UUID; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.env.Environment; +import org.springframework.transaction.annotation.Transactional; + +@SpringBootTest(properties = { + "spring.mail.enable=true" +}) +@Transactional +class ResetPasswordTokenServiceTestIT extends EmbeddedPostgreSQLTest { + + @RegisterExtension + static final GreenMailExtension greenMail = new GreenMailExtension(ServerSetupTest.SMTP) + .withConfiguration(GreenMailConfiguration.aConfig().withUser("compreface@exadel.com", "1234567890")) + .withPerMethodLifecycle(true); + + @Value("${forgot-password.email.subject}") + private String emailSubject; + + @Value("${forgot-password.email.message}") + private String emailMessage; + + @Autowired + private DbHelper dbHelper; + + @Autowired + private ResetPasswordTokenService tokenService; + + @Autowired + private ResetPasswordTokenRepository tokenRepository; + + @Autowired + private UserRepository userRepository; + + @Autowired + private Environment env; + + @Test + @SneakyThrows + void shouldAssignAndSendTokenWhenUserDoesNotHavaOne() { + var user = dbHelper.insertUser("john@gmail.com"); + + var tokensBefore = tokenRepository.findAll(); + var mailsBefore = greenMail.getReceivedMessages(); + + tokenService.assignAndSendToken(user.getEmail()); + + var tokensAfter = tokenRepository.findAll(); + var mailsAfter = greenMail.getReceivedMessages(); + + assertThat(tokensBefore).isEmpty(); + assertThat(mailsBefore).isEmpty(); + assertThat(tokensAfter).hasSize(1); + assertThat(mailsAfter).hasSize(1); + + var token = tokensAfter.get(0); + var mail = mailsAfter[0]; + + assertThat(token.getToken()).isNotNull(); + assertThat(token.getExpiresIn()).isAfter(now(UTC)); + assertThat(token.getUser()).isEqualTo(user); + + assertThat(mail.getAllRecipients()).hasSize(1); + assertThat(mail.getAllRecipients()[0].toString()).hasToString(user.getEmail()); + assertThat(mail.getSubject()).hasToString("CompreFace Reset Password"); + assertThat(GreenMailUtil.getBody(mail)).contains(buildMailBody(token.getToken())); + } + + @Test + @SneakyThrows + void shouldReassignAndSendTokenWhenUserHasOne() { + var user = dbHelper.insertUser("john@gmail.com"); + dbHelper.insertResetPasswordToken(user); + + var tokensBefore = tokenRepository.findAll(); + var mailsBefore = greenMail.getReceivedMessages(); + + tokenService.assignAndSendToken(user.getEmail()); + + var tokensAfter = tokenRepository.findAll(); + var mailsAfter = greenMail.getReceivedMessages(); + + assertThat(tokensBefore).hasSize(1); + assertThat(mailsBefore).isEmpty(); + assertThat(tokensAfter).hasSize(1); + assertThat(mailsAfter).hasSize(1); + + var tokenBefore = tokensBefore.get(0); + var tokenAfter = tokensAfter.get(0); + var mail = mailsAfter[0]; + + assertThat(tokenBefore.getToken()).isNotNull(); + assertThat(tokenBefore.getExpiresIn()).isAfter(now(UTC)); + assertThat(tokenBefore.getUser()).isEqualTo(user); + assertThat(tokenAfter.getToken()).isNotNull(); + assertThat(tokenAfter.getExpiresIn()).isAfter(now(UTC)); + assertThat(tokenAfter.getUser()).isEqualTo(user); + assertThat(tokenBefore.getToken()).isNotEqualByComparingTo(tokenAfter.getToken()); + + assertThat(mail.getAllRecipients()).hasSize(1); + assertThat(mail.getAllRecipients()[0].toString()).hasToString(user.getEmail()); + assertThat(mail.getSubject()).hasToString("CompreFace Reset Password"); + assertThat(GreenMailUtil.getBody(mail)).contains(buildMailBody(tokenAfter.getToken())); + } + + @Test + void shouldThrowUserDoesNotExistExceptionWhenThereIsNoActiveUserWithProvidedEmail() { + var tokensBefore = tokenRepository.findAll(); + var mailsBefore = greenMail.getReceivedMessages(); + + assertThatThrownBy(() -> tokenService.assignAndSendToken("john@gmail.com")) + .isInstanceOf(UserDoesNotExistException.class) + .hasMessageContaining("User john@gmail.com does not exist"); + + var tokensAfter = tokenRepository.findAll(); + var mailsAfter = greenMail.getReceivedMessages(); + + assertThat(tokensBefore).isEmpty(); + assertThat(mailsBefore).isEmpty(); + assertThat(tokensAfter).isEmpty(); + assertThat(mailsAfter).isEmpty(); + } + + @Test + void shouldReturnUserByTokenWhenBothExist() { + var expectedUser = dbHelper.insertUser("john@gmail.com"); + var expectedToken = dbHelper.insertResetPasswordToken(expectedUser); + + var tokensBefore = tokenRepository.findAll(); + + var actualUser = tokenService.exchangeTokenOnUser(expectedToken.getToken().toString()); + + var tokensAfter = tokenRepository.findAll(); + + assertThat(tokensBefore).hasSize(1); + assertThat(tokensAfter).isEmpty(); + + var actualToken = tokensBefore.get(0); + + assertThat(actualToken.getToken()).isEqualByComparingTo(expectedToken.getToken()); + assertThat(actualToken.getUser()).isEqualTo(expectedToken.getUser()); + assertThat(actualToken.getExpiresIn()).isEqualTo(expectedToken.getExpiresIn()); + + assertThat(actualUser.getEmail()).isEqualTo(expectedUser.getEmail()); + } + + @Test + void shouldThrowInvalidResetPasswordTokenExceptionWhenTokenDoesNotExist() { + var expectedUser = dbHelper.insertUser("john@gmail.com"); + var fakeToken = UUID.randomUUID().toString(); + + var tokensBefore = tokenRepository.findAll(); + + assertThatThrownBy(() -> tokenService.exchangeTokenOnUser(fakeToken)) + .isInstanceOf(InvalidResetPasswordTokenException.class) + .hasMessageContaining("The reset password token is invalid!"); + + var tokensAfter = tokenRepository.findAll(); + + assertThat(tokensBefore).isEmpty(); + assertThat(tokensAfter).isEmpty(); + + var actualUser = userRepository.findByEmailAndEnabledTrue(expectedUser.getEmail()).get(); + + assertThat(actualUser.getEmail()).isEqualTo(expectedUser.getEmail()); + } + + @Test + void shouldThrowInvalidResetPasswordTokenExceptionWhenInvalidTokenProvided() { + var invalidToken = "qqq-www-eee-rrr-ttt"; + + var tokensBefore = tokenRepository.findAll(); + + assertThatThrownBy(() -> tokenService.exchangeTokenOnUser(invalidToken)) + .isInstanceOf(InvalidResetPasswordTokenException.class) + .hasMessageContaining("The reset password token is invalid!"); + + var tokensAfter = tokenRepository.findAll(); + + assertThat(tokensBefore).isEmpty(); + assertThat(tokensAfter).isEmpty(); + } + + @Test + void shouldThrowInvalidResetPasswordTokenExceptionWhenExpiredTokenProvided() { + var user = dbHelper.insertUser("john@gmail.com"); + var expiredToken = dbHelper.insertResetPasswordToken(user, now(UTC).minus(1000, MILLIS)); + + var tokensBefore = tokenRepository.findAll(); + + assertThatThrownBy(() -> tokenService.exchangeTokenOnUser(expiredToken.getToken().toString())) + .isInstanceOf(InvalidResetPasswordTokenException.class) + .hasMessageContaining("The reset password token is invalid!"); + + var tokensAfter = tokenRepository.findAll(); + + assertThat(tokensBefore).hasSize(1); + assertThat(tokensAfter).hasSize(1); + } + + @Test + void shouldDeleteAllExpiredTokens() { + var user1 = dbHelper.insertUser("john@gmail.com"); + var user2 = dbHelper.insertUser("bob@gmail.com"); + var user3 = dbHelper.insertUser("alex@gmail.com"); + var user4 = dbHelper.insertUser("victor@gmail.com"); + + var token1 = dbHelper.insertResetPasswordToken(user1); + var token2 = dbHelper.insertResetPasswordToken(user2); + var expiredToken1 = dbHelper.insertResetPasswordToken(user3, now(UTC).minus(1000, MILLIS)); + var expiredToken2 = dbHelper.insertResetPasswordToken(user4, now(UTC).minus(1000, MILLIS)); + + var tokensBefore = tokenRepository.findAll(); + var usersBefore = userRepository.findAll(); + + tokenService.deleteExpiredTokens(); + + var tokensAfter = tokenRepository.findAll(); + var usersAfter = userRepository.findAll(); + + assertThat(tokensBefore).hasSize(4); + assertThat(tokensAfter).hasSize(2); + + assertThat(tokensBefore).contains(token1, token2, expiredToken1, expiredToken2); + assertThat(tokensAfter).contains(token1, token2); + assertThat(usersBefore).contains(user1, user2, user3, user4); + assertThat(usersAfter).contains(user1, user2, user3, user4); + } + + private String buildMailBody(UUID token) { + return String.format(emailMessage, env.getProperty("host.frs"), token.toString()); + } +} diff --git a/java/admin/src/test/java/com/exadel/frs/service/StatisticServiceTestIT.java b/java/admin/src/test/java/com/exadel/frs/service/StatisticServiceTestIT.java new file mode 100644 index 0000000000..2c3f9d7d12 --- /dev/null +++ b/java/admin/src/test/java/com/exadel/frs/service/StatisticServiceTestIT.java @@ -0,0 +1,150 @@ +package com.exadel.frs.service; + +import static com.exadel.frs.commonservice.enums.GlobalRole.OWNER; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import com.exadel.frs.DbHelper; +import com.exadel.frs.EmbeddedPostgreSQLTest; +import com.exadel.frs.commonservice.repository.InstallInfoRepository; +import com.exadel.frs.commonservice.repository.ModelRepository; +import com.exadel.frs.commonservice.repository.SubjectRepository; +import com.exadel.frs.commonservice.repository.UserRepository; +import com.exadel.frs.commonservice.system.feign.ApperyStatisticsClient; +import com.exadel.frs.commonservice.system.feign.StatisticsFacesEntity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.transaction.annotation.Transactional; + +@Transactional +@SpringBootTest +class StatisticServiceTestIT extends EmbeddedPostgreSQLTest { + + @Autowired + private DbHelper dbHelper; + + @Autowired + private UserRepository userRepository; + + @Autowired + private ModelRepository modelRepository; + + @Autowired + private SubjectRepository subjectRepository; + + @Autowired + private InstallInfoRepository installInfoRepository; + + @MockBean + private ApperyStatisticsClient apperyClient; + + @Autowired + private StatisticService statisticService; + + @BeforeEach + void cleanUp() { + userRepository.deleteAll(); + modelRepository.deleteAll(); + + userRepository.flush(); + modelRepository.flush(); + } + + @Test + void recordStatistics_UserDoesNotExist_ShouldExit() { + var model = dbHelper.insertModel(); + dbHelper.insertSubject(model, "Subject1"); + dbHelper.insertSubject(model, "Subject2"); + dbHelper.insertSubject(model, "Subject3"); + + statisticService.recordStatistics(); + + assertThat(userRepository.count()).isZero(); + assertThat(modelRepository.count()).isEqualTo(1); + assertThat(subjectRepository.count()).isEqualTo(3); + + verifyNoInteractions(apperyClient); + } + + @Test + void recordStatistics_UserIsNotOwner_ShouldExit() { + dbHelper.insertUser("john@gmail.com"); + + var model = dbHelper.insertModel(); + dbHelper.insertSubject(model, "Subject1"); + dbHelper.insertSubject(model, "Subject2"); + dbHelper.insertSubject(model, "Subject3"); + + statisticService.recordStatistics(); + + assertThat(userRepository.count()).isEqualTo(1); + assertThat(modelRepository.count()).isEqualTo(1); + assertThat(subjectRepository.count()).isEqualTo(3); + + verifyNoInteractions(apperyClient); + } + + @Test + void recordStatistics_InstallInfoDoesNotExist_ShouldExit() { + installInfoRepository.deleteAll(); + installInfoRepository.flush(); + + dbHelper.insertUser("john@gmail.com"); + + var model = dbHelper.insertModel(); + dbHelper.insertSubject(model, "Subject1"); + dbHelper.insertSubject(model, "Subject2"); + dbHelper.insertSubject(model, "Subject3"); + + statisticService.recordStatistics(); + + assertThat(userRepository.count()).isEqualTo(1); + assertThat(modelRepository.count()).isEqualTo(1); + assertThat(subjectRepository.count()).isEqualTo(3); + assertThat(installInfoRepository.count()).isZero(); + + verifyNoInteractions(apperyClient); + } + + @Test + void recordStatistics_ThereIsOneModelThatHasThreeSubjects_ShouldRecordStatisticWithRangeBetweenOneAndTen() { + dbHelper.insertUser("john@gmail.com", OWNER); + + var model = dbHelper.insertModel(); + dbHelper.insertSubject(model, "Subject1"); + dbHelper.insertSubject(model, "Subject2"); + dbHelper.insertSubject(model, "Subject3"); + + var installGuid = installInfoRepository.findTopByOrderByInstallGuid().getInstallGuid(); + + statisticService.recordStatistics(); + + assertThat(userRepository.count()).isEqualTo(1); + assertThat(modelRepository.count()).isEqualTo(1); + assertThat(subjectRepository.count()).isEqualTo(3); + + var statistic = new StatisticsFacesEntity(installGuid, model.getGuid(), "1-10"); + + verify(apperyClient).create("qwe1-rty2-uio3", statistic); + } + + @Test + void recordStatistics_SomethingWentWrongWhileSendingStatistics_ShouldThrowApperyServiceException() { + dbHelper.insertUser("john@gmail.com", OWNER); + + var model = dbHelper.insertModel(); + dbHelper.insertSubject(model, "Subject1"); + dbHelper.insertSubject(model, "Subject2"); + dbHelper.insertSubject(model, "Subject3"); + + assertThat(userRepository.count()).isEqualTo(1); + assertThat(modelRepository.count()).isEqualTo(1); + assertThat(subjectRepository.count()).isEqualTo(3); + } +} diff --git a/java/admin/src/test/java/com/exadel/frs/UserServiceTest.java b/java/admin/src/test/java/com/exadel/frs/service/UserServiceTest.java similarity index 98% rename from java/admin/src/test/java/com/exadel/frs/UserServiceTest.java rename to java/admin/src/test/java/com/exadel/frs/service/UserServiceTest.java index e92ff821d9..e3924dcbd5 100644 --- a/java/admin/src/test/java/com/exadel/frs/UserServiceTest.java +++ b/java/admin/src/test/java/com/exadel/frs/service/UserServiceTest.java @@ -14,7 +14,7 @@ * permissions and limitations under the License. */ -package com.exadel.frs; +package com.exadel.frs.service; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -28,22 +28,21 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; - -import com.exadel.frs.commonservice.exception.EmptyRequiredFieldException; -import com.exadel.frs.dto.ui.UserCreateDto; -import com.exadel.frs.dto.ui.UserDeleteDto; -import com.exadel.frs.dto.ui.UserUpdateDto; import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.enums.GlobalRole; import com.exadel.frs.commonservice.enums.Replacer; -import com.exadel.frs.exception.EmailAlreadyRegisteredException; +import com.exadel.frs.commonservice.exception.EmptyRequiredFieldException; import com.exadel.frs.commonservice.exception.IllegalReplacerException; +import com.exadel.frs.commonservice.repository.UserRepository; +import com.exadel.frs.dto.UserCreateDto; +import com.exadel.frs.dto.UserDeleteDto; +import com.exadel.frs.dto.UserUpdateDto; +import com.exadel.frs.exception.EmailAlreadyRegisteredException; import com.exadel.frs.exception.IncorrectUserPasswordException; import com.exadel.frs.exception.InvalidEmailException; import com.exadel.frs.exception.RegistrationTokenExpiredException; import com.exadel.frs.exception.UserDoesNotExistException; import com.exadel.frs.helpers.EmailSender; -import com.exadel.frs.commonservice.repository.UserRepository; import com.exadel.frs.service.AppService; import com.exadel.frs.service.UserService; import com.exadel.frs.system.security.AuthorizationManager; @@ -116,6 +115,7 @@ void failGetUser() { @Test void successCreateUserWhenMailServerEnabled() { when(env.getProperty("spring.mail.enable")).thenReturn("true"); + when(env.getProperty("host.frs")).thenReturn("http://localhost"); when(userRepositoryMock.save(any())).thenAnswer(returnsFirstArg()); val userCreateDto = UserCreateDto.builder() .email("email@example.com") @@ -273,6 +273,7 @@ void confirmRegistrationReturns403WhenTokenIsExpired() { void confirmRegistrationEnablesUserAndRemovesTokenWhenSuccess() { when(userRepositoryMock.save(any())).thenAnswer(returnsFirstArg()); when(env.getProperty("spring.mail.enable")).thenReturn("true"); + when(env.getProperty("host.frs")).thenReturn("http://localhost"); val userCreateDto = UserCreateDto.builder() .email("email@example.com") .password("password") diff --git a/java/admin/src/test/java/com/exadel/frs/UserServiceTestIT.java b/java/admin/src/test/java/com/exadel/frs/service/UserServiceTestIT.java similarity index 76% rename from java/admin/src/test/java/com/exadel/frs/UserServiceTestIT.java rename to java/admin/src/test/java/com/exadel/frs/service/UserServiceTestIT.java index 322491847a..88467ef77b 100644 --- a/java/admin/src/test/java/com/exadel/frs/UserServiceTestIT.java +++ b/java/admin/src/test/java/com/exadel/frs/service/UserServiceTestIT.java @@ -14,11 +14,13 @@ * permissions and limitations under the License. */ -package com.exadel.frs; +package com.exadel.frs.service; +import com.exadel.frs.DbHelper; +import com.exadel.frs.EmbeddedPostgreSQLTest; import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.repository.UserRepository; -import com.exadel.frs.dto.ui.UserCreateDto; +import com.exadel.frs.dto.UserCreateDto; import com.exadel.frs.exception.UserDoesNotExistException; import com.exadel.frs.helpers.EmailSender; import com.exadel.frs.service.UserService; @@ -36,9 +38,14 @@ import java.util.Optional; import java.util.UUID; +import org.springframework.transaction.annotation.Transactional; +import static com.exadel.frs.commonservice.enums.GlobalRole.OWNER; +import static com.exadel.frs.commonservice.enums.GlobalRole.USER; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; @ExtendWith(SpringExtension.class) @@ -60,6 +67,9 @@ class UserServiceTestIT extends EmbeddedPostgreSQLTest { @SpyBean private UserService userService; + @Autowired + private DbHelper dbHelper; + @Autowired private UserRepository userRepository; @@ -147,6 +157,34 @@ void autocompleteReturnsUsers() { assertThat(actual).hasSize(2); } + @Test + @Transactional + void confirmRegistration_ThereAreTwoUnconfirmedUsers_FirstOfThemShouldBecomeAnOwner() { + userRepository.deleteAll(); + + val user1 = dbHelper.insertUnconfirmedUser(USER_EMAIL); + val user2 = dbHelper.insertUnconfirmedUser(USER_EMAIL_2); + + assertFalse(user1.isEnabled()); + assertFalse(user2.isEnabled()); + assertThat(user1.getRegistrationToken()).isNotNull(); + assertThat(user2.getRegistrationToken()).isNotNull(); + assertThat(user1.getGlobalRole()).isEqualByComparingTo(USER); + assertThat(user2.getGlobalRole()).isEqualByComparingTo(USER); + assertThat(userRepository.findAll()).containsOnly(user1, user2); + + userService.confirmRegistration(user1.getRegistrationToken()); + userService.confirmRegistration(user2.getRegistrationToken()); + + assertTrue(user1.isEnabled()); + assertTrue(user2.isEnabled()); + assertThat(user1.getRegistrationToken()).isNull(); + assertThat(user2.getRegistrationToken()).isNull(); + assertThat(user1.getGlobalRole()).isEqualByComparingTo(OWNER); + assertThat(user2.getGlobalRole()).isEqualByComparingTo(USER); + assertThat(userRepository.findAll()).containsOnly(user1, user2); + } + private void createAndEnableUser(final String email) { val regToken = UUID.randomUUID().toString(); when(userService.generateRegistrationToken()).thenReturn(regToken); diff --git a/java/admin/src/test/java/com/exadel/frs/system/security/AuthorizationManagerTest.java b/java/admin/src/test/java/com/exadel/frs/system/security/AuthorizationManagerTest.java index ee32a6377f..2e5ddd4707 100644 --- a/java/admin/src/test/java/com/exadel/frs/system/security/AuthorizationManagerTest.java +++ b/java/admin/src/test/java/com/exadel/frs/system/security/AuthorizationManagerTest.java @@ -26,7 +26,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -import com.exadel.frs.dto.ui.UserDeleteDto; +import com.exadel.frs.dto.UserDeleteDto; import com.exadel.frs.commonservice.entity.App; import com.exadel.frs.commonservice.entity.Model; import com.exadel.frs.commonservice.entity.User; diff --git a/java/admin/src/test/java/com/exadel/frs/system/security/endpoint/CustomTokenEndpointTest.java b/java/admin/src/test/java/com/exadel/frs/system/security/endpoint/CustomTokenEndpointTest.java new file mode 100644 index 0000000000..e059decd5c --- /dev/null +++ b/java/admin/src/test/java/com/exadel/frs/system/security/endpoint/CustomTokenEndpointTest.java @@ -0,0 +1,209 @@ +package com.exadel.frs.system.security.endpoint; + +import static com.exadel.frs.system.global.Constants.ADMIN; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.cookie; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import com.exadel.frs.EmbeddedPostgreSQLTest; +import com.exadel.frs.FrsApplication; +import com.exadel.frs.commonservice.entity.User; +import com.exadel.frs.commonservice.repository.UserRepository; +import com.exadel.frs.service.UserService; +import com.fasterxml.jackson.databind.ObjectMapper; +import javax.annotation.PostConstruct; +import javax.servlet.http.Cookie; +import lombok.SneakyThrows; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.security.web.FilterChainProxy; +import org.springframework.test.context.web.WebAppConfiguration; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +@WebAppConfiguration +@SpringBootTest(classes = FrsApplication.class) +class CustomTokenEndpointTest extends EmbeddedPostgreSQLTest { + + @Value("${spring.mail.enable}") + private boolean isMailEnabled; + + @Autowired + private WebApplicationContext wac; + + @Autowired + private FilterChainProxy springSecurityFilterChain; + + @Autowired + private UserRepository userRepository; + + @SpyBean + private UserService userService; + + private MockMvc mockMvc; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @PostConstruct + void setup() { + mockMvc = MockMvcBuilders + .webAppContextSetup(wac) + .addFilter(springSecurityFilterChain) + .build(); + } + + @BeforeEach + @SneakyThrows + void beforeEach() { + when(userService.generateRegistrationToken()).thenReturn("MockRegistrationToken"); + when(userService.hasOnlyDemoUser()).thenReturn(false); + + registerMockUser(); + } + + @AfterEach + void afterEach() { + userRepository.deleteAll(); + } + + @Test + @SneakyThrows + void shouldReturnAccessTokenCookieAndRefreshTokenCookieWhenUserSignIn() { + + mockMvc.perform(post(ADMIN + "/oauth/token") + .param("grant_type", "password") + .param("scope", "all") + .param("username", "mockuser@gmail.com") + .param("password", "password") + .with(httpBasic("CommonClientId", "password")) + .accept("application/json;charset=UTF-8")) + .andExpect(status().isOk()) + .andExpect(cookie().exists("CFSESSION")) + .andExpect(cookie().exists("REFRESH_TOKEN")) + .andDo(print()); + } + + @Test + @SneakyThrows + void shouldReturnBadRequestStatusCodeWhenUserCredentialsAreInvalid() { + + mockMvc.perform(post(ADMIN + "/oauth/token") + .param("grant_type", "password") + .param("scope", "all") + .param("username", "invaliduser@gmail.com") + .param("password", "password") + .with(httpBasic("CommonClientId", "password")) + .accept("application/json;charset=UTF-8")) + .andExpect(status().isBadRequest()) + .andExpect(cookie().doesNotExist("CFSESSION")) + .andExpect(cookie().doesNotExist("REFRESH_TOKEN")) + .andDo(print()); + } + + @Test + @SneakyThrows + void shouldReturnUnauthorizedStatusCodeWhenClientCredentialsAreInvalid() { + + mockMvc.perform(post(ADMIN + "/oauth/token") + .param("grant_type", "password") + .param("scope", "all") + .param("username", "mockuser@gmail.com") + .param("password", "password") + .with(httpBasic("InvalidClientId", "password")) + .accept("application/json;charset=UTF-8")) + .andExpect(status().isUnauthorized()) + .andExpect(cookie().doesNotExist("CFSESSION")) + .andExpect(cookie().doesNotExist("REFRESH_TOKEN")) + .andDo(print()); + } + + @Test + @SneakyThrows + void shouldReturnNewAccessTokenAndNewRefreshTokenWhenRefreshTokenRequestOccurred() { + + var signInResult = mockMvc.perform(post(ADMIN + "/oauth/token") + .param("grant_type", "password") + .param("scope", "all") + .param("username", "mockuser@gmail.com") + .param("password", "password") + .with(httpBasic("CommonClientId", "password")) + .accept("application/json;charset=UTF-8")) + .andExpect(status().isOk()) + .andExpect(cookie().exists("CFSESSION")) + .andExpect(cookie().exists("REFRESH_TOKEN")) + .andDo(print()); + + var accessTokenCookie = signInResult.andReturn().getResponse().getCookie("CFSESSION"); + var refreshTokenCookie = signInResult.andReturn().getResponse().getCookie("REFRESH_TOKEN"); + + var refreshTokenResult = mockMvc.perform(post(ADMIN + "/oauth/token") + .param("grant_type", "refresh_token") + .param("scope", "all") + .with(httpBasic("CommonClientId", "password")) + .accept("application/json;charset=UTF-8") + .cookie(refreshTokenCookie)) + .andExpect(status().isOk()) + .andExpect(cookie().exists("CFSESSION")) + .andExpect(cookie().exists("REFRESH_TOKEN")) + .andDo(print()); + + var newAccessTokenCookie = refreshTokenResult.andReturn().getResponse().getCookie("CFSESSION"); + var newRefreshTokenCookie = refreshTokenResult.andReturn().getResponse().getCookie("REFRESH_TOKEN"); + + assertThat(accessTokenCookie.getValue()).isNotEqualTo(newAccessTokenCookie.getValue()); + assertThat(refreshTokenCookie.getValue()).isNotEqualTo(newRefreshTokenCookie.getValue()); + } + + @Test + @SneakyThrows + void shouldReturnBadRequestStatusCodeWhenRefreshTokenRequestOccurredWithInvalidRefreshToken() { + + var refreshTokenCookie = new Cookie("REFRESH_TOKEN", "InvalidValue"); + + refreshTokenCookie.setHttpOnly(true); + refreshTokenCookie.setPath(ADMIN + "/oauth/token"); + refreshTokenCookie.setMaxAge(-1); + + mockMvc.perform(post(ADMIN + "/oauth/token") + .param("grant_type", "refresh_token") + .param("scope", "all") + .with(httpBasic("CommonClientId", "password")) + .accept("application/json;charset=UTF-8") + .cookie(refreshTokenCookie)) + .andExpect(status().isBadRequest()) + .andExpect(cookie().doesNotExist("CFSESSION")) + .andExpect(cookie().doesNotExist("REFRESH_TOKEN")) + .andDo(print()); + } + + @SneakyThrows + private void registerMockUser() { + + var mockUser = User.builder() + .email("mockuser@gmail.com") + .firstName("firstName") + .lastName("lastName") + .password("password") + .build(); + + mockMvc.perform(post(ADMIN + "/user/register") + .contentType("application/json") + .accept("application/json") + .content(objectMapper.writeValueAsString(mockUser))) + .andExpect(status().isCreated()) + .andDo(print()); + + if (isMailEnabled) { + userService.confirmRegistration("MockRegistrationToken"); + } + } +} diff --git a/java/admin/src/test/java/com/exadel/frs/validation/ValidatorTest.java b/java/admin/src/test/java/com/exadel/frs/validation/ValidatorTest.java index ca84af4421..1b17f8cc82 100644 --- a/java/admin/src/test/java/com/exadel/frs/validation/ValidatorTest.java +++ b/java/admin/src/test/java/com/exadel/frs/validation/ValidatorTest.java @@ -18,7 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import com.exadel.frs.EmbeddedPostgreSQLTest; -import com.exadel.frs.dto.ui.UserRoleUpdateDto; +import com.exadel.frs.dto.UserRoleUpdateDto; import javax.validation.Validator; import lombok.val; import org.junit.jupiter.api.Test; diff --git a/java/admin/src/test/resources/application.yml b/java/admin/src/test/resources/application.yml index 7b0eb04ca8..2fdaa1516c 100644 --- a/java/admin/src/test/resources/application.yml +++ b/java/admin/src/test/resources/application.yml @@ -17,20 +17,20 @@ app: feign: appery-io: url: https://localhost/rest/1/db/collections - api-key: ${APPERY_API_KEY:#{null}} + api-key: ${APPERY_API_KEY:qwe1-rty2-uio3} + faces: + connect-timeout: ${CONNECTION_TIMEOUT:10000} + read-timeout: ${READ_TIMEOUT:60000} + retryer: + max-attempts: ${MAX_ATTEMPTS:1} spring: servlet: multipart: - max-file-size: 10MB - max-request-size: 10MB + max-file-size: ${MAX_FILE_SIZE:5MB} + max-request-size: ${MAX_REQUEST_SIZE:10MB} flyway: enabled: false - datasource: - driver-class-name: org.postgresql.Driver - url: ${POSTGRES_URL:jdbc:postgresql://compreface-postgres-db:5432/frs} - username: ${POSTGRES_USER:postgres} - password: ${POSTGRES_PASSWORD:postgres} jpa: properties: hibernate: @@ -43,11 +43,22 @@ spring: database: postgresql open-in-view: true generate-ddl: false + liquibase: + parameters: + common-client: + client-id: ${app.security.oauth2.clients.COMMON.client-id} + access-token-validity: ${app.security.oauth2.clients.COMMON.access-token-validity} + refresh-token-validity: ${app.security.oauth2.clients.COMMON.refresh-token-validity} + authorized-grant-types: ${app.security.oauth2.clients.COMMON.authorized-grant-types} mail: - enable: ${ENABLE_EMAIL_SERVER:false} - host: ${EMAIL_HOST:example.com} - from: ${EMAIL_FROM:name } + from: compreface@exadel.com + username: compreface@exadel.com + password: 1234567890 + host: 127.0.0.1 + protocol: smtp + port: 3025 test-connection: false + enable: false properties.mail: debug: true smtp: @@ -78,6 +89,13 @@ registration: scheduler: period: 300000 +forgot-password: + reset-password-token: + expires: 900000 + cleaner: + scheduler: + cron: '0 0 0 * * ?' + environment: servers: PYTHON: @@ -94,4 +112,8 @@ image: - ico - gif - webp - saveImagesToDB: ${SAVE_IMAGES_TO_DB:true} \ No newline at end of file + saveImagesToDB: ${SAVE_IMAGES_TO_DB:true} + +statistic: + model: + months: ${MODEL_STATISTIC_MONTHS:6} diff --git a/java/api/pom.xml b/java/api/pom.xml index 2adfdf898d..d16a2ce60b 100644 --- a/java/api/pom.xml +++ b/java/api/pom.xml @@ -143,7 +143,11 @@ com.impossibl.pgjdbc-ng pgjdbc-ng - ${pgjdbc-ng.version} + + + com.cronutils + cron-utils + ${cron-utils.version} org.springframework.boot @@ -162,6 +166,10 @@ org.apache.maven.plugins maven-compiler-plugin + + org.apache.maven.plugins + maven-surefire-plugin + org.springframework.boot spring-boot-maven-plugin @@ -169,4 +177,4 @@ - \ No newline at end of file + diff --git a/java/api/src/main/java/com/exadel/frs/TrainServiceApplication.java b/java/api/src/main/java/com/exadel/frs/TrainServiceApplication.java index 0daa7d5d80..4fce901217 100644 --- a/java/api/src/main/java/com/exadel/frs/TrainServiceApplication.java +++ b/java/api/src/main/java/com/exadel/frs/TrainServiceApplication.java @@ -20,7 +20,9 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableScheduling; +@EnableScheduling @EnableFeignClients(basePackages = "com.exadel.frs.commonservice.system.feign") @SpringBootApplication(exclude = {DataSourceAutoConfiguration.class}) public class TrainServiceApplication { @@ -28,4 +30,4 @@ public class TrainServiceApplication { public static void main(String[] args) { SpringApplication.run(TrainServiceApplication.class, args); } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/aspect/MigrationWriteControlAspect.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/aspect/MigrationWriteControlAspect.java index 6bbdcde97a..83c6bbb465 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/aspect/MigrationWriteControlAspect.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/aspect/MigrationWriteControlAspect.java @@ -51,4 +51,4 @@ public Object writeEndpoint(final ProceedingJoinPoint pjp) throws Throwable { return pjp.proceed(); } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/aspect/WriteEndpoint.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/aspect/WriteEndpoint.java index 6ebc247309..c80c09c27b 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/aspect/WriteEndpoint.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/aspect/WriteEndpoint.java @@ -25,4 +25,4 @@ @Retention(RetentionPolicy.RUNTIME) public @interface WriteEndpoint { -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/EmbeddingCacheProvider.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/EmbeddingCacheProvider.java index f8e38978ea..294d52fac7 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/EmbeddingCacheProvider.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/EmbeddingCacheProvider.java @@ -38,7 +38,7 @@ public EmbeddingCollection getOrLoad(final String apiKey) { var result = cache.getIfPresent(apiKey); if (result == null) { - result = embeddingService.doWithEmbeddingsStream(apiKey, EmbeddingCollection::from); + result = embeddingService.doWithEnhancedEmbeddingProjectionStream(apiKey, EmbeddingCollection::from); cache.put(apiKey, result); @@ -52,7 +52,7 @@ public void ifPresent(String apiKey, Consumer consumer) { Optional.ofNullable(cache.getIfPresent(apiKey)) .ifPresent(consumer); - EmbeddingCollection dd = cache.getIfPresent(apiKey); + cache.getIfPresent(apiKey); notifyCacheEvent("UPDATE", apiKey); } @@ -63,7 +63,7 @@ public void invalidate(final String apiKey) { public void receivePutOnCache(String apiKey) { - var result = embeddingService.doWithEmbeddingsStream(apiKey, EmbeddingCollection::from); + var result = embeddingService.doWithEnhancedEmbeddingProjectionStream(apiKey, EmbeddingCollection::from); cache.put(apiKey, result); } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/EmbeddingCollection.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/EmbeddingCollection.java index 0c66433769..7ab38b91b8 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/EmbeddingCollection.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/EmbeddingCollection.java @@ -1,36 +1,44 @@ package com.exadel.frs.core.trainservice.cache; import com.exadel.frs.commonservice.entity.Embedding; -import com.exadel.frs.commonservice.entity.EmbeddingProjection; +import com.exadel.frs.commonservice.exception.IncorrectImageIdException; +import com.exadel.frs.commonservice.projection.EmbeddingProjection; +import com.exadel.frs.commonservice.projection.EnhancedEmbeddingProjection; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Stream; import lombok.AccessLevel; import lombok.AllArgsConstructor; +import lombok.val; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.indexing.NDArrayIndex; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @AllArgsConstructor(access = AccessLevel.PRIVATE) public class EmbeddingCollection { private final BiMap projection2Index; private INDArray embeddings; - public static EmbeddingCollection from(final Stream embeddings) { - final var rawEmbeddings = new LinkedList(); - final Map projections2Index = new HashMap<>(); + public static EmbeddingCollection from(final Stream stream) { + val rawEmbeddings = new LinkedList(); + val projections2Index = new HashMap(); + val index = new AtomicInteger(); // just to bypass 'final' variables restriction inside lambdas - var index = new AtomicInteger(); // just to bypass 'final' variables restriction inside lambdas - - embeddings.forEach(embedding -> { - rawEmbeddings.add(embedding.getEmbedding()); - projections2Index.put(EmbeddingProjection.from(embedding), index.getAndIncrement()); + stream.forEach(projection -> { + projections2Index.put(EmbeddingProjection.from(projection), index.getAndIncrement()); + rawEmbeddings.add(projection.embeddingData()); }); return new EmbeddingCollection( @@ -67,8 +75,8 @@ public INDArray getEmbeddings() { public synchronized void updateSubjectName(String oldSubjectName, String newSubjectName) { final List projections = projection2Index.keySet() .stream() - .filter(projection -> projection.getSubjectName().equals(oldSubjectName)) - .collect(Collectors.toList()); + .filter(projection -> projection.subjectName().equals(oldSubjectName)) + .toList(); projections.forEach(projection -> projection2Index.put( projection.withNewSubjectName(newSubjectName), @@ -97,8 +105,8 @@ public synchronized Collection removeEmbeddingsBySubjectNam // not efficient at ALL! review current approach! final List toRemove = projection2Index.keySet().stream() - .filter(projection -> projection.getSubjectName().equals(subjectName)) - .collect(Collectors.toList()); + .filter(projection -> projection.subjectName().equals(subjectName)) + .toList(); toRemove.forEach(this::removeEmbedding); // <- rethink @@ -138,26 +146,31 @@ public synchronized Optional getRawEmbeddingById(UUID embeddingId) { return findByEmbeddingId( embeddingId, // return duplicated row - entry -> embeddings.getRow(entry.getValue()).dup() + entry -> embeddings.getRow(entry.getValue(), true).dup() ); } public synchronized Optional getSubjectNameByEmbeddingId(UUID embeddingId) { return findByEmbeddingId( embeddingId, - entry -> entry.getKey().getSubjectName() + entry -> entry.getKey().subjectName() ); } private Optional findByEmbeddingId(UUID embeddingId, Function, T> func) { - if (embeddingId == null) { - return Optional.empty(); - } + validImageId(embeddingId); - return projection2Index.entrySet() + return Optional.ofNullable(projection2Index.entrySet() .stream() - .filter(entry -> embeddingId.equals(entry.getKey().getEmbeddingId())) + .filter(entry -> embeddingId.equals(entry.getKey().embeddingId())) .findFirst() - .map(func); + .map(func) + .orElseThrow(IncorrectImageIdException::new)); + } + + private void validImageId(UUID embeddingId) { + if (embeddingId == null) { + throw new IncorrectImageIdException(); + } } } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/ModelStatisticCacheProvider.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/ModelStatisticCacheProvider.java new file mode 100644 index 0000000000..1604ed9257 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/cache/ModelStatisticCacheProvider.java @@ -0,0 +1,29 @@ +package com.exadel.frs.core.trainservice.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.util.HashMap; +import java.util.Map; +import org.springframework.stereotype.Component; + +@Component +public class ModelStatisticCacheProvider { + + private static final Cache cache = CacheBuilder.newBuilder().build(); + + public void incrementRequestCount(final long key) { + cache.asMap().compute(key, (k, v) -> v == null ? 1 : v + 1); + } + + public Map getCacheCopyAsMap() { + return new HashMap<>(cache.asMap()); + } + + public boolean isEmpty() { + return cache.size() == 0; + } + + public void invalidateCache() { + cache.invalidateAll(); + } +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/FaceClassifierPredictor.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/FaceClassifierPredictor.java index bef3b593c0..cc2fcb0cb2 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/FaceClassifierPredictor.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/FaceClassifierPredictor.java @@ -41,4 +41,4 @@ public Double verify(final String modelKey, final double[] input, final UUID emb public double[] verify(final double[] sourceImageEmbedding, final double[][] targetImageEmbedding) { return classifier.verify(sourceImageEmbedding, targetImageEmbedding); } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/classifiers/Classifier.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/classifiers/Classifier.java index e27e69f0b5..ebed573a19 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/classifiers/Classifier.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/classifiers/Classifier.java @@ -29,4 +29,4 @@ public interface Classifier extends Serializable { Double verify(double[] input, String apiKey, UUID embeddingId); double[] verify(double[] sourceImageEmbedding, double[][] targetImageEmbedding); -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/classifiers/EuclideanDistanceClassifier.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/classifiers/EuclideanDistanceClassifier.java index b5ae91b5c3..518ea84a0c 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/classifiers/EuclideanDistanceClassifier.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/classifiers/EuclideanDistanceClassifier.java @@ -64,7 +64,7 @@ public List> predict(final double[] input, final String api var prob = probabilities[sortedIndexes[i]]; var embedding = indexMap.get(sortedIndexes[i]); - result.add(Pair.of(prob, embedding.getSubjectName())); + result.add(Pair.of(prob, embedding.subjectName())); } } return result; @@ -139,6 +139,7 @@ private INDArray calculateSimilarities(INDArray distance) { } List coefficients = status.getSimilarityCoefficients(); + // (tanh ((coef0 - distance) * coef1) + 1) / 2 return Transforms.tanh(distance.rsubi(coefficients.get(0)).muli(coefficients.get(1)), false).addi(1).divi(2); } @@ -161,4 +162,4 @@ private static int[] sortedIndexes(double[] probabilities) { .mapToInt(index -> index) .toArray(); } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/migration/MigrationComponent.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/migration/MigrationComponent.java index b5151ceb8b..274942eb87 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/migration/MigrationComponent.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/migration/MigrationComponent.java @@ -87,7 +87,8 @@ private Optional recalculate(UUID embeddingId, byte[] content) { new MultipartFileData(content, "recalculated", null), 1, null, - CALCULATOR_PLUGIN + CALCULATOR_PLUGIN, + true ); return findFacesResponse.getResult().stream() @@ -100,4 +101,4 @@ private Optional recalculate(UUID embeddingId, byte[] content) { return Optional.empty(); } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/migration/MigrationStatusStorage.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/migration/MigrationStatusStorage.java index 5066a8593c..e14fcac65d 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/component/migration/MigrationStatusStorage.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/component/migration/MigrationStatusStorage.java @@ -45,4 +45,4 @@ public void finishMigration() { public boolean isMigrating() { return isMigrating.get(); } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/config/AspectConfiguration.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/config/AspectConfiguration.java index 3baf86754f..fd984e785d 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/config/AspectConfiguration.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/config/AspectConfiguration.java @@ -23,4 +23,4 @@ @EnableAspectJAutoProxy public class AspectConfiguration { -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/config/AsyncConfiguration.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/config/AsyncConfiguration.java index ca16f81ac0..4b0f19852f 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/config/AsyncConfiguration.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/config/AsyncConfiguration.java @@ -23,4 +23,4 @@ @EnableAsync public class AsyncConfiguration { -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/config/SpringFoxConfig.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/config/SpringFoxConfig.java index ac59bc2eb8..0efeddf463 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/config/SpringFoxConfig.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/config/SpringFoxConfig.java @@ -37,4 +37,3 @@ public Docket api() { .build(); } } - diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/config/repository/NotificationDbConfig.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/config/repository/NotificationDbConfig.java index 140d72f3f2..c070df74f4 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/config/repository/NotificationDbConfig.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/config/repository/NotificationDbConfig.java @@ -1,31 +1,23 @@ package com.exadel.frs.core.trainservice.config.repository; import com.impossibl.postgres.jdbc.PGDataSource; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; +import org.springframework.context.annotation.Profile; +@Profile("!test") @Configuration public class NotificationDbConfig { - @Autowired - private Environment env; - @Bean(name = "dsPgNot") - public PGDataSource pgNotificationDatasource() { - PGDataSource dataSource = new PGDataSource(); - - String dbUrl = env.getProperty("spring.datasource-pg.url"); - String dbUsername = env.getProperty("spring.datasource-pg.username"); - String dbPassword = env.getProperty("spring.datasource-pg.password"); + public PGDataSource pgNotificationDatasource(DataSourceProperties dataSourceProperties) { + String pgsqlUrl = dataSourceProperties.getUrl().replace("postgresql", "pgsql"); + dataSourceProperties.setUrl(pgsqlUrl); - String databaseUrl = dbUrl.replaceAll("postgresql", "pgsql"); - - dataSource.setDatabaseUrl(databaseUrl); - dataSource.setUser(dbUsername); - dataSource.setPassword(dbPassword); + PGDataSource dataSource = dataSourceProperties.initializeDataSourceBuilder().type(PGDataSource.class).build(); dataSource.setHousekeeper(false); + return dataSource; } } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/ConfigController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/ConfigController.java new file mode 100644 index 0000000000..da59ac83cb --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/ConfigController.java @@ -0,0 +1,36 @@ +package com.exadel.frs.core.trainservice.controller; + +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import com.exadel.frs.core.trainservice.dto.ConfigDto; +import io.swagger.annotations.ApiOperation; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.springframework.core.env.Environment; +import org.springframework.util.unit.DataSize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping(API_V1 + "/config") +@RequiredArgsConstructor +public class ConfigController { + + private final Environment env; + + @GetMapping + @ApiOperation(value = "Returns configuration properties of the application") + public ConfigDto getConfig() { + return ConfigDto.builder() + .maxFileSize(getNumericPropertyAsBytes("spring.servlet.multipart.max-file-size")) + .maxBodySize(getNumericPropertyAsBytes("spring.servlet.multipart.max-request-size")) + .build(); + } + + private Long getNumericPropertyAsBytes(String propertyName) { + return Optional.ofNullable(env.getProperty(propertyName)) + .map(DataSize::parse) + .map(DataSize::toBytes) + .orElse(null); + } +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/ConsistenceController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/ConsistenceController.java index 7c2e68dce2..dc81102c32 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/ConsistenceController.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/ConsistenceController.java @@ -1,5 +1,7 @@ package com.exadel.frs.core.trainservice.controller; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.commonservice.enums.AppStatus.OK; import com.exadel.frs.commonservice.sdk.faces.FacesApiClient; import com.exadel.frs.commonservice.system.global.ImageProperties; import com.exadel.frs.core.trainservice.dto.VersionConsistenceDto; @@ -9,8 +11,6 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; - @RestController @RequestMapping(API_V1 + "/consistence") @RequiredArgsConstructor @@ -29,6 +29,7 @@ public VersionConsistenceDto getCheckDemo() { .demoFaceCollectionIsInconsistent(embeddingService.isDemoCollectionInconsistent()) .dbIsInconsistent(embeddingService.isDbInconsistent(calculatorVersion)) .saveImagesToDB(imageProperties.isSaveImagesToDB()) + .status(OK) .build(); } } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/DetectionController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/DetectionController.java index eb2e8ddc6b..569f73074d 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/DetectionController.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/DetectionController.java @@ -16,6 +16,20 @@ package com.exadel.frs.core.trainservice.controller; +import static com.exadel.frs.commonservice.system.global.Constants.DET_PROB_THRESHOLD; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.core.trainservice.system.global.Constants.DET_PROB_THRESHOLD_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.FACE_PLUGINS; +import static com.exadel.frs.core.trainservice.system.global.Constants.FACE_PLUGINS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_FILE_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_MIN_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.NUMBER_VALUE_EXAMPLE; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.X_FRS_API_KEY_HEADER; import com.exadel.frs.core.trainservice.dto.Base64File; import com.exadel.frs.core.trainservice.dto.FacesDetectionResponseDto; import com.exadel.frs.core.trainservice.dto.ProcessImageParams; @@ -23,18 +37,18 @@ import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParams; import io.swagger.annotations.ApiParam; +import javax.validation.Valid; +import javax.validation.constraints.Min; import lombok.RequiredArgsConstructor; import org.springframework.http.MediaType; import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import javax.validation.Valid; -import javax.validation.constraints.Min; - -import static com.exadel.frs.commonservice.system.global.Constants.DET_PROB_THRESHOLD; -import static com.exadel.frs.core.trainservice.system.global.Constants.*; - @RestController @RequestMapping(API_V1) @RequiredArgsConstructor @@ -53,11 +67,22 @@ public class DetectionController { required = true) }) public FacesDetectionResponseDto detect( - @ApiParam(value = IMAGE_FILE_DESC, required = true) @RequestParam final MultipartFile file, - @ApiParam(value = LIMIT_DESC) @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) @Min(value = 0, message = LIMIT_MIN_DESC) final Integer limit, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, - @ApiParam(value = FACE_PLUGINS_DESC) @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") final String facePlugins, - @ApiParam(value = STATUS_DESC) @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) final Boolean status + @ApiParam(value = IMAGE_FILE_DESC, required = true) + @RequestParam + final MultipartFile file, + @ApiParam(value = LIMIT_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) + @Min(value = 0, message = LIMIT_MIN_DESC) + final Integer limit, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, + @ApiParam(value = FACE_PLUGINS_DESC) + @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") + final String facePlugins, + @ApiParam(value = STATUS_DESC) + @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) + final Boolean status ) { var processImageParams = ProcessImageParams .builder() @@ -81,11 +106,23 @@ public FacesDetectionResponseDto detect( required = true) }) public FacesDetectionResponseDto detectBase64( - @ApiParam(value = LIMIT_DESC) @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) @Min(value = 0, message = LIMIT_MIN_DESC) final Integer limit, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, - @ApiParam(value = FACE_PLUGINS_DESC) @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") final String facePlugins, - @ApiParam(value = STATUS_DESC) @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) final Boolean status, - @Valid @RequestBody Base64File request) { + @ApiParam(value = LIMIT_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) + @Min(value = 0, message = LIMIT_MIN_DESC) + final Integer limit, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, + @ApiParam(value = FACE_PLUGINS_DESC) + @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") + final String facePlugins, + @ApiParam(value = STATUS_DESC) + @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) + final Boolean status, + @Valid + @RequestBody + final Base64File request + ) { var processImageParams = ProcessImageParams .builder() .imageBase64(request.getContent()) @@ -97,4 +134,4 @@ public FacesDetectionResponseDto detectBase64( return (FacesDetectionResponseDto) detectionService.processImage(processImageParams); } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/EmbeddingController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/EmbeddingController.java index 81c361fd1f..45e01db8e2 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/EmbeddingController.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/EmbeddingController.java @@ -1,11 +1,39 @@ package com.exadel.frs.core.trainservice.controller; - +import static com.exadel.frs.commonservice.system.global.Constants.DET_PROB_THRESHOLD; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_KEY_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.core.trainservice.system.global.Constants.CACHE_CONTROL_HEADER_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.DET_PROB_THRESHOLD_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.FACE_PLUGINS; +import static com.exadel.frs.core.trainservice.system.global.Constants.FACE_PLUGINS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_ID; +import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_IDS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_ID_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_WITH_ONE_FACE_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_MIN_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.NUMBER_VALUE_EXAMPLE; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.SUBJECT; +import static com.exadel.frs.core.trainservice.system.global.Constants.SUBJECT_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.SUBJECT_NAME_IS_EMPTY; +import static com.exadel.frs.core.trainservice.system.global.Constants.X_FRS_API_KEY_HEADER; +import static org.springframework.http.HttpStatus.CREATED; import com.exadel.frs.commonservice.entity.Embedding; import com.exadel.frs.commonservice.entity.Img; import com.exadel.frs.commonservice.entity.Subject; import com.exadel.frs.core.trainservice.aspect.WriteEndpoint; -import com.exadel.frs.core.trainservice.dto.*; +import com.exadel.frs.core.trainservice.dto.Base64File; +import com.exadel.frs.core.trainservice.dto.EmbeddingDto; +import com.exadel.frs.core.trainservice.dto.EmbeddingsRecognitionRequest; +import com.exadel.frs.core.trainservice.dto.EmbeddingsVerificationProcessResponse; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; +import com.exadel.frs.core.trainservice.dto.ProcessImageParams; +import com.exadel.frs.core.trainservice.dto.VerificationResult; import com.exadel.frs.core.trainservice.mapper.EmbeddingMapper; import com.exadel.frs.core.trainservice.mapper.FacesMapper; import com.exadel.frs.core.trainservice.service.EmbeddingService; @@ -13,28 +41,37 @@ import com.exadel.frs.core.trainservice.validation.ImageExtensionValidator; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiParam; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotBlank; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.Pair; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.*; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import javax.validation.Valid; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotEmpty; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static com.exadel.frs.commonservice.system.global.Constants.DET_PROB_THRESHOLD; -import static com.exadel.frs.core.trainservice.system.global.Constants.*; -import static org.springframework.http.HttpStatus.CREATED; - +@Validated @RestController -@RequestMapping(API_V1 + "/recognition/faces") +@RequestMapping(API_V1 + "/recognition") @RequiredArgsConstructor public class EmbeddingController { @@ -46,12 +83,22 @@ public class EmbeddingController { @WriteEndpoint @ResponseStatus(CREATED) - @PostMapping + @PostMapping("/faces") public EmbeddingDto addEmbedding( - @ApiParam(value = IMAGE_WITH_ONE_FACE_DESC, required = true) @RequestParam final MultipartFile file, - @ApiParam(value = SUBJECT_DESC, required = true) @RequestParam(SUBJECT) final String subjectName, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey + @ApiParam(value = IMAGE_WITH_ONE_FACE_DESC, required = true) + @RequestParam + final MultipartFile file, + @ApiParam(value = SUBJECT_DESC, required = true) + @Valid + @NotBlank(message = SUBJECT_NAME_IS_EMPTY) + @RequestParam(SUBJECT) + final String subjectName, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey ) throws IOException { imageValidator.validate(file); @@ -67,12 +114,23 @@ public EmbeddingDto addEmbedding( @WriteEndpoint @ResponseStatus(CREATED) - @PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE) + @PostMapping(value = "/faces", consumes = MediaType.APPLICATION_JSON_VALUE) public EmbeddingDto addEmbeddingBase64( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = SUBJECT_DESC) @RequestParam(value = SUBJECT) String subjectName, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, - @Valid @RequestBody Base64File request) { + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = SUBJECT_DESC) + @Valid + @NotBlank(message = SUBJECT_NAME_IS_EMPTY) + @RequestParam(value = SUBJECT) + final String subjectName, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, + @Valid + @RequestBody + final Base64File request + ) { imageValidator.validateBase64(request.getContent()); final Pair pair = subjectService.saveCalculatedEmbedding( @@ -85,28 +143,47 @@ public EmbeddingDto addEmbeddingBase64( return new EmbeddingDto(pair.getRight().getId().toString(), subjectName); } - @GetMapping(value = "/{embeddingId}/img", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) - public @ResponseBody - byte[] downloadImg( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(name = X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = IMAGE_ID_DESC, required = true) @PathVariable final UUID embeddingId) { + @ResponseBody + @GetMapping(value = "/faces/{embeddingId}/img", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) + public byte[] downloadImg(HttpServletResponse response, + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(name = X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = IMAGE_ID_DESC, required = true) + @PathVariable + final UUID embeddingId + ) { + response.addHeader(HttpHeaders.CACHE_CONTROL, CACHE_CONTROL_HEADER_VALUE); return embeddingService.getImg(apiKey, embeddingId) - .map(Img::getContent) - .orElse(new byte[]{}); + .map(Img::getContent) + .orElse(new byte[]{}); } - @GetMapping + @GetMapping("/faces") public Faces listEmbeddings( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(name = X_FRS_API_KEY_HEADER) final String apiKey, - Pageable pageable) { - return new Faces(embeddingService.listEmbeddings(apiKey, pageable).map(embeddingMapper::toResponseDto)); + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(name = X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = SUBJECT_DESC) + @Valid + @RequestParam(name = SUBJECT, required = false) + final String subjectName, + final Pageable pageable + ) { + return new Faces(embeddingService.listEmbeddings(apiKey, subjectName, pageable).map(embeddingMapper::toResponseDto)); } @WriteEndpoint - @DeleteMapping + @DeleteMapping("/faces") public Map removeAllSubjectEmbeddings( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(name = X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = SUBJECT_DESC) @RequestParam( name = SUBJECT, required = false) final String subjectName + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(name = X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = SUBJECT_DESC) + @Validated + @NotBlank(message = SUBJECT_NAME_IS_EMPTY) + @RequestParam(name = SUBJECT, required = false) + final String subjectName ) { return Map.of( "deleted", @@ -115,34 +192,71 @@ public Map removeAllSubjectEmbeddings( } @WriteEndpoint - @DeleteMapping("/{embeddingId}") + @DeleteMapping("/faces/{embeddingId}") public EmbeddingDto deleteEmbeddingById( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(name = X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = IMAGE_ID_DESC, required = true) @PathVariable final UUID embeddingId) { + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(name = X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = IMAGE_ID_DESC, required = true) + @PathVariable + final UUID embeddingId + ) { var embedding = subjectService.removeSubjectEmbedding(apiKey, embeddingId); return new EmbeddingDto(embeddingId.toString(), embedding.getSubject().getSubjectName()); } - @PostMapping(value = "/{embeddingId}/verify", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @WriteEndpoint + @PostMapping("/faces/delete") + public List deleteEmbeddingsById( + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(name = X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = IMAGE_IDS_DESC, required = true) + @RequestBody + final List embeddingIds + ) { + List list = subjectService.removeSubjectEmbeddings(apiKey, embeddingIds); + List dtoList = list.stream() + .map(c -> new EmbeddingDto(c.getId().toString(), c.getSubject().getSubjectName())) + .toList(); + return dtoList; + } + + @PostMapping(value = "/faces/{embeddingId}/verify", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public VerificationResult recognizeFile( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = IMAGE_WITH_ONE_FACE_DESC, required = true) @RequestParam final MultipartFile file, - @ApiParam(value = LIMIT_DESC) @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) @Min(value = 0, message = LIMIT_MIN_DESC) final Integer limit, - @ApiParam(value = IMAGE_ID_DESC, required = true) @PathVariable final UUID embeddingId, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, - @ApiParam(value = FACE_PLUGINS_DESC) @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") final String facePlugins, - @ApiParam(value = STATUS_DESC) @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) final Boolean status + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = IMAGE_WITH_ONE_FACE_DESC, required = true) + @RequestParam + final MultipartFile file, + @ApiParam(value = LIMIT_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) + @Min(value = 0, message = LIMIT_MIN_DESC) + final Integer limit, + @ApiParam(value = IMAGE_ID_DESC, required = true) + @PathVariable + final UUID embeddingId, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, + @ApiParam(value = FACE_PLUGINS_DESC) + @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") + final String facePlugins, + @ApiParam(value = STATUS_DESC) + @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) + final Boolean status ) { imageValidator.validate(file); var processImageParams = ProcessImageParams.builder() - .additionalParams(Map.of(IMAGE_ID, embeddingId)) - .apiKey(apiKey) - .detProbThreshold(detProbThreshold) - .file(file) - .facePlugins(facePlugins) - .limit(limit) - .status(status) - .build(); + .additionalParams(Map.of(IMAGE_ID, embeddingId)) + .apiKey(apiKey) + .detProbThreshold(detProbThreshold) + .file(file) + .facePlugins(facePlugins) + .limit(limit) + .status(status) + .build(); var pair = subjectService.verifyFace(processImageParams); return new VerificationResult( @@ -151,27 +265,42 @@ public VerificationResult recognizeFile( ); } - @PostMapping(value = "/{embeddingId}/verify", consumes = MediaType.APPLICATION_JSON_VALUE) + @PostMapping(value = "/faces/{embeddingId}/verify", consumes = MediaType.APPLICATION_JSON_VALUE) public VerificationResult recognizeBase64( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = IMAGE_ID_DESC, required = true) @PathVariable final UUID embeddingId, - @ApiParam(value = LIMIT_DESC) @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) @Min(value = 0, message = LIMIT_MIN_DESC) final Integer limit, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, - @ApiParam(value = FACE_PLUGINS_DESC) @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") final String facePlugins, - @ApiParam(value = STATUS_DESC) @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) final Boolean status, - @RequestBody @Valid Base64File request + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = IMAGE_ID_DESC, required = true) + @PathVariable + final UUID embeddingId, + @ApiParam(value = LIMIT_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) + @Min(value = 0, message = LIMIT_MIN_DESC) + final Integer limit, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, + @ApiParam(value = FACE_PLUGINS_DESC) + @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") + final String facePlugins, + @ApiParam(value = STATUS_DESC) + @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) + final Boolean status, + @RequestBody + @Valid + final Base64File request ) { imageValidator.validateBase64(request.getContent()); var processImageParams = ProcessImageParams.builder() - .additionalParams(Map.of(IMAGE_ID, embeddingId)) - .apiKey(apiKey) - .detProbThreshold(detProbThreshold) - .imageBase64(request.getContent()) - .facePlugins(facePlugins) - .limit(limit) - .status(status) - .build(); + .additionalParams(Map.of(IMAGE_ID, embeddingId)) + .apiKey(apiKey) + .detProbThreshold(detProbThreshold) + .imageBase64(request.getContent()) + .facePlugins(facePlugins) + .limit(limit) + .status(status) + .build(); var pair = subjectService.verifyFace(processImageParams); return new VerificationResult( @@ -180,6 +309,28 @@ public VerificationResult recognizeBase64( ); } + @PostMapping(value = "/embeddings/faces/{imageId}/verify", consumes = MediaType.APPLICATION_JSON_VALUE) + public EmbeddingsVerificationProcessResponse recognizeEmbeddings( + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = IMAGE_ID_DESC, required = true) + @PathVariable + final UUID imageId, + @RequestBody + @Valid + final EmbeddingsRecognitionRequest recognitionRequest + ) { + ProcessEmbeddingsParams processParams = + ProcessEmbeddingsParams.builder() + .apiKey(apiKey) + .embeddings(recognitionRequest.getEmbeddings()) + .additionalParams(Map.of(IMAGE_ID, imageId)) + .build(); + + return subjectService.verifyEmbedding(processParams); + } + @RequiredArgsConstructor private static final class Faces { diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/MigrateController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/MigrateController.java index eab1cbb91c..ef73aa2b68 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/MigrateController.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/MigrateController.java @@ -16,6 +16,7 @@ package com.exadel.frs.core.trainservice.controller; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; import com.exadel.frs.core.trainservice.component.migration.MigrationComponent; import com.exadel.frs.core.trainservice.component.migration.MigrationStatusStorage; import lombok.RequiredArgsConstructor; @@ -25,8 +26,6 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; - @RestController @RequestMapping(API_V1) @RequiredArgsConstructor diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/RecognizeController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/RecognizeController.java index 8c0da7015f..b629f8702f 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/RecognizeController.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/RecognizeController.java @@ -16,50 +16,87 @@ package com.exadel.frs.core.trainservice.controller; +import static com.exadel.frs.commonservice.system.global.Constants.DET_PROB_THRESHOLD; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_KEY_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.core.trainservice.system.global.Constants.DETECT_FACES; +import static com.exadel.frs.core.trainservice.system.global.Constants.DETECT_FACES_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.DETECT_FACES_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.DET_PROB_THRESHOLD_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.FACE_PLUGINS; +import static com.exadel.frs.core.trainservice.system.global.Constants.FACE_PLUGINS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_FILE_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_MIN_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.NUMBER_VALUE_EXAMPLE; +import static com.exadel.frs.core.trainservice.system.global.Constants.PREDICTION_COUNT; +import static com.exadel.frs.core.trainservice.system.global.Constants.PREDICTION_COUNT_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.PREDICTION_COUNT_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.PREDICTION_COUNT_MIN_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.PREDICTION_COUNT_REQUEST_PARAM; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.X_FRS_API_KEY_HEADER; import com.exadel.frs.core.trainservice.dto.Base64File; +import com.exadel.frs.core.trainservice.dto.EmbeddingsRecognitionProcessResponse; +import com.exadel.frs.core.trainservice.dto.EmbeddingsRecognitionRequest; import com.exadel.frs.core.trainservice.dto.FacesRecognitionResponseDto; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; import com.exadel.frs.core.trainservice.dto.ProcessImageParams; -import com.exadel.frs.core.trainservice.service.FaceProcessService; +import com.exadel.frs.core.trainservice.service.EmbeddingsProcessService; import io.swagger.annotations.ApiParam; +import java.util.Collections; +import javax.validation.Valid; +import javax.validation.constraints.Min; import lombok.RequiredArgsConstructor; import org.springframework.http.MediaType; import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import javax.validation.Valid; -import javax.validation.constraints.Min; -import java.util.Collections; - -import static com.exadel.frs.commonservice.system.global.Constants.DET_PROB_THRESHOLD; -import static com.exadel.frs.core.trainservice.system.global.Constants.*; - @RestController @RequestMapping(API_V1) @RequiredArgsConstructor @Validated public class RecognizeController { - private final FaceProcessService recognitionService; + private final EmbeddingsProcessService recognitionService; @PostMapping(value = "/recognition/recognize", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public FacesRecognitionResponseDto recognize( @ApiParam(value = API_KEY_DESC, required = true) - @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, @ApiParam(value = IMAGE_FILE_DESC, required = true) - @RequestParam final MultipartFile file, - @ApiParam(value = LIMIT_DESC) + @RequestParam + final MultipartFile file, + @ApiParam(value = LIMIT_DESC, example = NUMBER_VALUE_EXAMPLE) @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) - @Min(value = 0, message = LIMIT_MIN_DESC) final Integer limit, - @ApiParam(value = PREDICTION_COUNT_DESC) + @Min(value = 0, message = LIMIT_MIN_DESC) + final Integer limit, + @ApiParam(value = PREDICTION_COUNT_DESC, example = NUMBER_VALUE_EXAMPLE) @RequestParam(defaultValue = PREDICTION_COUNT_DEFAULT_VALUE, name = PREDICTION_COUNT_REQUEST_PARAM, required = false) - @Min(value = 1, message = PREDICTION_COUNT_MIN_DESC) final Integer predictionCount, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) - @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, + @Min(value = 1, message = PREDICTION_COUNT_MIN_DESC) + final Integer predictionCount, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, @ApiParam(value = FACE_PLUGINS_DESC) - @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") final String facePlugins, + @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") + final String facePlugins, @ApiParam(value = STATUS_DESC) - @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) final Boolean status + @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) + final Boolean status, + @ApiParam(value = DETECT_FACES_DESC) + @RequestParam(value = DETECT_FACES, required = false, defaultValue = DETECT_FACES_DEFAULT_VALUE) + final Boolean detectFaces ) { ProcessImageParams processImageParams = ProcessImageParams .builder() @@ -69,6 +106,7 @@ public FacesRecognitionResponseDto recognize( .detProbThreshold(detProbThreshold) .facePlugins(facePlugins) .status(status) + .detectFaces(detectFaces) .additionalParams(Collections.singletonMap(PREDICTION_COUNT, predictionCount)) .build(); @@ -77,14 +115,33 @@ public FacesRecognitionResponseDto recognize( @PostMapping(value = "/recognition/recognize", consumes = MediaType.APPLICATION_JSON_VALUE) public FacesRecognitionResponseDto recognizeBase64( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = LIMIT_DESC) @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) @Min(value = 0, message = LIMIT_MIN_DESC) final Integer limit, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, - @ApiParam(value = FACE_PLUGINS_DESC) @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") final String facePlugins, - @ApiParam(value = STATUS_DESC) @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) final Boolean status, - @ApiParam(value = PREDICTION_COUNT_DESC) @RequestParam(value = PREDICTION_COUNT_REQUEST_PARAM, required = false, defaultValue = PREDICTION_COUNT_DEFAULT_VALUE) @Min(value = 1, message = PREDICTION_COUNT_MIN_DESC) Integer predictionCount, - @RequestBody @Valid Base64File request) { - + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = LIMIT_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) + @Min(value = 0, message = LIMIT_MIN_DESC) + final Integer limit, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, + @ApiParam(value = FACE_PLUGINS_DESC) + @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") + final String facePlugins, + @ApiParam(value = STATUS_DESC) + @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) + final Boolean status, + @ApiParam(value = DETECT_FACES_DESC) + @RequestParam(value = DETECT_FACES, required = false, defaultValue = DETECT_FACES_DEFAULT_VALUE) + final Boolean detectFaces, + @ApiParam(value = PREDICTION_COUNT_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = PREDICTION_COUNT_REQUEST_PARAM, required = false, defaultValue = PREDICTION_COUNT_DEFAULT_VALUE) + @Min(value = 1, message = PREDICTION_COUNT_MIN_DESC) + final Integer predictionCount, + @RequestBody + @Valid + final Base64File request + ) { ProcessImageParams processImageParams = ProcessImageParams .builder() .apiKey(apiKey) @@ -93,9 +150,33 @@ public FacesRecognitionResponseDto recognizeBase64( .detProbThreshold(detProbThreshold) .facePlugins(facePlugins) .status(status) + .detectFaces(detectFaces) .additionalParams(Collections.singletonMap(PREDICTION_COUNT, predictionCount)) .build(); return (FacesRecognitionResponseDto) recognitionService.processImage(processImageParams); } -} \ No newline at end of file + + @PostMapping(value = "/recognition/embeddings/recognize", consumes = MediaType.APPLICATION_JSON_VALUE) + public EmbeddingsRecognitionProcessResponse recognizeEmbeddings( + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = PREDICTION_COUNT_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = PREDICTION_COUNT_REQUEST_PARAM, required = false, defaultValue = PREDICTION_COUNT_DEFAULT_VALUE) + @Min(value = 1, message = PREDICTION_COUNT_MIN_DESC) + final Integer predictionCount, + @RequestBody + @Valid + final EmbeddingsRecognitionRequest recognitionRequest + ) { + ProcessEmbeddingsParams processParams = + ProcessEmbeddingsParams.builder() + .apiKey(apiKey) + .embeddings(recognitionRequest.getEmbeddings()) + .additionalParams(Collections.singletonMap(PREDICTION_COUNT, predictionCount)) + .build(); + + return (EmbeddingsRecognitionProcessResponse) recognitionService.processEmbeddings(processParams); + } +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/StaticController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/StaticController.java index b4981c9486..58ac171d5e 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/StaticController.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/StaticController.java @@ -1,15 +1,22 @@ package com.exadel.frs.core.trainservice.controller; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_KEY_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.core.trainservice.system.global.Constants.CACHE_CONTROL_HEADER_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_ID_DESC; import com.exadel.frs.commonservice.entity.Img; import com.exadel.frs.core.trainservice.service.EmbeddingService; import io.swagger.annotations.ApiParam; +import java.util.UUID; +import javax.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.*; - -import java.util.UUID; - -import static com.exadel.frs.core.trainservice.system.global.Constants.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; @RestController @RequestMapping(API_V1 + "/static") @@ -18,13 +25,20 @@ public class StaticController { private final EmbeddingService embeddingService; + @ResponseBody @GetMapping(value = "/{apiKey}/images/{embeddingId}", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) - public @ResponseBody - byte[] downloadImg( - @ApiParam(value = API_KEY_DESC, required = true) @PathVariable("apiKey") final String apiKey, - @ApiParam(value = IMAGE_ID_DESC, required = true) @PathVariable final UUID embeddingId) { + public byte[] downloadImg( + @ApiParam(value = API_KEY_DESC, required = true) + @PathVariable("apiKey") + final String apiKey, + @ApiParam(value = IMAGE_ID_DESC, required = true) + @PathVariable + final UUID embeddingId, + final HttpServletResponse response + ) { + response.addHeader(HttpHeaders.CACHE_CONTROL, CACHE_CONTROL_HEADER_VALUE); return embeddingService.getImg(apiKey, embeddingId) - .map(Img::getContent) - .orElse(new byte[]{}); + .map(Img::getContent) + .orElse(new byte[]{}); } } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/SubjectController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/SubjectController.java index b0d7aba8da..f9f41a4222 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/SubjectController.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/SubjectController.java @@ -1,18 +1,31 @@ package com.exadel.frs.core.trainservice.controller; - +import static com.exadel.frs.core.trainservice.system.global.Constants.API_KEY_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.core.trainservice.system.global.Constants.SUBJECT_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.SUBJECT_NAME_IS_EMPTY; +import static com.exadel.frs.core.trainservice.system.global.Constants.X_FRS_API_KEY_HEADER; +import static org.springframework.http.HttpStatus.CREATED; import com.exadel.frs.core.trainservice.dto.SubjectDto; import com.exadel.frs.core.trainservice.service.SubjectService; import io.swagger.annotations.ApiParam; -import lombok.RequiredArgsConstructor; -import org.springframework.web.bind.annotation.*; - -import javax.validation.Valid; import java.util.Map; +import javax.validation.Valid; +import javax.validation.constraints.NotBlank; +import lombok.RequiredArgsConstructor; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; -import static com.exadel.frs.core.trainservice.system.global.Constants.*; -import static org.springframework.http.HttpStatus.CREATED; - +@Validated @RestController @RequestMapping(API_V1 + "/recognition/subjects") @RequiredArgsConstructor @@ -23,15 +36,23 @@ public class SubjectController { @PostMapping @ResponseStatus(CREATED) public SubjectDto createSubject( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, - @Valid @RequestBody final SubjectDto subjectDto) { + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @Valid + @RequestBody + final SubjectDto subjectDto + ) { var subject = subjectService.createSubject(apiKey, subjectDto.getSubjectName()); return new SubjectDto((subject.getSubjectName())); } @GetMapping public Map listSubjects( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey) { + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey + ) { return Map.of( "subjects", subjectService.getSubjectsNames(apiKey) @@ -40,9 +61,17 @@ public Map listSubjects( @PutMapping("/{subject}") public Map renameSubject( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = SUBJECT_DESC, required = true) @PathVariable("subject") final String oldSubjectName, - @Valid @RequestBody final SubjectDto subjectDto) { + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = SUBJECT_DESC, required = true) + @Valid + @NotBlank(message = SUBJECT_NAME_IS_EMPTY) + @PathVariable("subject") + final String oldSubjectName, + @Valid + @RequestBody + final SubjectDto subjectDto) { return Map.of( "updated", subjectService.updateSubjectName(apiKey, oldSubjectName, subjectDto.getSubjectName()) @@ -50,15 +79,30 @@ public Map renameSubject( } @DeleteMapping("/{subject}") - public void deleteSubject( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = SUBJECT_DESC, required = true) @PathVariable("subject") final String subjectName) { - subjectService.deleteSubjectByName(apiKey, subjectName); + public Map deleteSubject( + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = SUBJECT_DESC, required = true) + @Valid + @NotBlank(message = SUBJECT_NAME_IS_EMPTY) + @PathVariable("subject") + final String subjectName + ) { + return Map.of( + "subject", + subjectService.deleteSubjectByName(apiKey, subjectName) + .getRight() + .getSubjectName() + ); } @DeleteMapping public Map deleteSubjects( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey) { + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey + ) { return Map.of( "deleted", subjectService.deleteSubjectsByApiKey(apiKey) diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/VerifyController.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/VerifyController.java index 92e81fb4b4..dd00252610 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/VerifyController.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/controller/VerifyController.java @@ -1,51 +1,82 @@ package com.exadel.frs.core.trainservice.controller; - +import static com.exadel.frs.commonservice.system.global.Constants.DET_PROB_THRESHOLD; +import static com.exadel.frs.core.trainservice.service.FaceVerificationProcessServiceImpl.RESULT; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_KEY_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.core.trainservice.system.global.Constants.DET_PROB_THRESHOLD_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.FACE_PLUGINS; +import static com.exadel.frs.core.trainservice.system.global.Constants.FACE_PLUGINS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.LIMIT_MIN_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.NUMBER_VALUE_EXAMPLE; +import static com.exadel.frs.core.trainservice.system.global.Constants.SOURCE_IMAGE; +import static com.exadel.frs.core.trainservice.system.global.Constants.SOURCE_IMAGE_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS_DEFAULT_VALUE; +import static com.exadel.frs.core.trainservice.system.global.Constants.STATUS_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.TARGET_IMAGE; +import static com.exadel.frs.core.trainservice.system.global.Constants.TARGET_IMAGE_DESC; +import static com.exadel.frs.core.trainservice.system.global.Constants.X_FRS_API_KEY_HEADER; +import com.exadel.frs.core.trainservice.dto.EmbeddingsVerificationProcessResponse; +import com.exadel.frs.core.trainservice.dto.EmbeddingsVerificationRequest; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; import com.exadel.frs.core.trainservice.dto.ProcessImageParams; import com.exadel.frs.core.trainservice.dto.VerifyFacesResponse; import com.exadel.frs.core.trainservice.dto.VerifySourceTargetRequest; +import com.exadel.frs.core.trainservice.service.EmbeddingsProcessService; import com.exadel.frs.core.trainservice.service.FaceProcessService; import io.swagger.annotations.ApiParam; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import javax.validation.Valid; +import javax.validation.constraints.Min; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.ArrayUtils; import org.springframework.http.MediaType; import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import javax.validation.Valid; -import javax.validation.constraints.Min; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static com.exadel.frs.commonservice.system.global.Constants.DET_PROB_THRESHOLD; -import static com.exadel.frs.core.trainservice.service.FaceVerificationProcessServiceImpl.RESULT; -import static com.exadel.frs.core.trainservice.system.global.Constants.*; - @RestController @RequestMapping(API_V1) @RequiredArgsConstructor @Validated public class VerifyController { - private final FaceProcessService verificationService; + private final EmbeddingsProcessService verificationService; @PostMapping(value = "/verification/verify", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public Map> verify( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, @ApiParam(value = SOURCE_IMAGE_DESC, required = true) - @RequestParam(name = SOURCE_IMAGE) final MultipartFile sourceImage, + @RequestParam(name = SOURCE_IMAGE) + final MultipartFile sourceImage, @ApiParam(value = TARGET_IMAGE_DESC, required = true) - @RequestParam(name = TARGET_IMAGE) final MultipartFile targetImage, - @ApiParam(value = LIMIT_DESC) + @RequestParam(name = TARGET_IMAGE) + final MultipartFile targetImage, + @ApiParam(value = LIMIT_DESC, example = NUMBER_VALUE_EXAMPLE) @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) - @Min(value = 0, message = LIMIT_MIN_DESC) final Integer limit, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) - @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, + @Min(value = 0, message = LIMIT_MIN_DESC) + final Integer limit, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, @ApiParam(value = FACE_PLUGINS_DESC) - @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") final String facePlugins, + @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") + final String facePlugins, @ApiParam(value = STATUS_DESC) - @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) final Boolean status + @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) + final Boolean status ) { Map fileMap = Map.of( SOURCE_IMAGE, sourceImage, @@ -68,13 +99,26 @@ public Map> verify( @PostMapping(value = "/verification/verify", consumes = MediaType.APPLICATION_JSON_VALUE) public Map> verifyBase64( - @ApiParam(value = API_KEY_DESC, required = true) @RequestHeader(X_FRS_API_KEY_HEADER) final String apiKey, - @ApiParam(value = LIMIT_DESC) @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) @Min(value = 0, message = LIMIT_MIN_DESC) final Integer limit, - @ApiParam(value = DET_PROB_THRESHOLD_DESC) @RequestParam(value = DET_PROB_THRESHOLD, required = false) final Double detProbThreshold, - @ApiParam(value = FACE_PLUGINS_DESC) @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") final String facePlugins, - @ApiParam(value = STATUS_DESC) @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) final Boolean status, - @RequestBody @Valid VerifySourceTargetRequest request) { - + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @ApiParam(value = LIMIT_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(defaultValue = LIMIT_DEFAULT_VALUE, required = false) + @Min(value = 0, message = LIMIT_MIN_DESC) + final Integer limit, + @ApiParam(value = DET_PROB_THRESHOLD_DESC, example = NUMBER_VALUE_EXAMPLE) + @RequestParam(value = DET_PROB_THRESHOLD, required = false) + final Double detProbThreshold, + @ApiParam(value = FACE_PLUGINS_DESC) + @RequestParam(value = FACE_PLUGINS, required = false, defaultValue = "") + final String facePlugins, + @ApiParam(value = STATUS_DESC) + @RequestParam(value = STATUS, required = false, defaultValue = STATUS_DEFAULT_VALUE) + final Boolean status, + @RequestBody + @Valid + final VerifySourceTargetRequest request + ) { Map fileMap = Map.of( SOURCE_IMAGE, request.getSourceImageBase64(), TARGET_IMAGE, request.getTargetImageBase64() @@ -93,4 +137,25 @@ public Map> verifyBase64( final VerifyFacesResponse response = (VerifyFacesResponse) verificationService.processImage(processImageParams); return Map.of(RESULT, Collections.singletonList(response)); } + + @PostMapping(value = "/verification/embeddings/verify", consumes = MediaType.APPLICATION_JSON_VALUE) + public EmbeddingsVerificationProcessResponse verifyEmbeddings( + @ApiParam(value = API_KEY_DESC, required = true) + @RequestHeader(X_FRS_API_KEY_HEADER) + final String apiKey, + @RequestBody + @Valid + final EmbeddingsVerificationRequest verificationRequest + ) { + double[] source = verificationRequest.getSource(); + double[][] targets = verificationRequest.getTargets(); + + ProcessEmbeddingsParams processParams = + ProcessEmbeddingsParams.builder() + .apiKey(apiKey) + .embeddings(ArrayUtils.insert(0, targets, source)) + .build(); + + return (EmbeddingsVerificationProcessResponse) verificationService.processEmbeddings(processParams); + } } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dao/SubjectDao.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dao/SubjectDao.java index aad9878246..f470241262 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dao/SubjectDao.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dao/SubjectDao.java @@ -9,6 +9,7 @@ import com.exadel.frs.commonservice.repository.EmbeddingRepository; import com.exadel.frs.commonservice.repository.ImgRepository; import com.exadel.frs.commonservice.repository.SubjectRepository; +import com.exadel.frs.commonservice.system.global.ImageProperties; import com.exadel.frs.core.trainservice.dto.EmbeddingInfo; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.Pair; @@ -25,6 +26,7 @@ public class SubjectDao { private final SubjectRepository subjectRepository; private final EmbeddingRepository embeddingRepository; private final ImgRepository imgRepository; + private final ImageProperties imageProperties; public Collection getSubjectNames(final String apiKey) { return subjectRepository.getSubjectNames(apiKey); @@ -158,8 +160,8 @@ public Pair addEmbedding(final String apiKey, final @Nullable EmbeddingInfo embeddingInfo) { var subject = subjectRepository - .findByApiKeyAndSubjectNameIgnoreCase(apiKey, subjectName) // subject already exists - .orElseGet(() -> saveSubject(apiKey, subjectName)); // add new subject + .findByApiKeyAndSubjectNameIgnoreCase(apiKey, subjectName) // subject already exists + .orElseGet(() -> saveSubject(apiKey, subjectName)); // add new subject Embedding embedding = null; if (embeddingInfo != null) { @@ -178,19 +180,16 @@ private Subject saveSubject(String apiKey, String subjectName) { } private Embedding saveEmbeddingInfo(Subject subject, EmbeddingInfo embeddingInfo) { - Img img = null; - if (embeddingInfo.getSource() != null) { - img = new Img(); - img.setContent(embeddingInfo.getSource()); - - imgRepository.save(img); - } - var embedding = new Embedding(); embedding.setSubject(subject); embedding.setEmbedding(embeddingInfo.getEmbedding()); embedding.setCalculator(embeddingInfo.getCalculator()); - embedding.setImg(img); + if (embeddingInfo.getSource() != null && imageProperties.isSaveImagesToDB()) { + Img img = new Img(); + img.setContent(embeddingInfo.getSource()); + imgRepository.save(img); + embedding.setImg(img); + } return embeddingRepository.save(embedding); } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/CacheActionDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/CacheActionDto.java index 8288a3b64a..d15d444c25 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/CacheActionDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/CacheActionDto.java @@ -9,10 +9,13 @@ @AllArgsConstructor @NoArgsConstructor public class CacheActionDto { + @JsonProperty("cacheAction") private String cacheAction; + @JsonProperty("apiKey") private String apiKey; + @JsonProperty("uuid") private String serverUUID; - } +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ConfigDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ConfigDto.java new file mode 100644 index 0000000000..4796db8bf8 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ConfigDto.java @@ -0,0 +1,20 @@ +package com.exadel.frs.core.trainservice.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ConfigDto { + + @JsonProperty("clientMaxFileSize") + private Long maxFileSize; + + @JsonProperty("clientMaxBodySize") + private Long maxBodySize; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingDto.java index a818522a19..b05122596a 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingDto.java @@ -35,4 +35,4 @@ public class EmbeddingDto { @JsonProperty("subject") private String subjectName; -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingRecognitionProcessResult.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingRecognitionProcessResult.java new file mode 100644 index 0000000000..7659eb8d7c --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingRecognitionProcessResult.java @@ -0,0 +1,17 @@ +package com.exadel.frs.core.trainservice.dto; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EmbeddingRecognitionProcessResult { + + private double[] embedding; + private List similarities; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingSimilarityResult.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingSimilarityResult.java new file mode 100644 index 0000000000..6d95df65a7 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingSimilarityResult.java @@ -0,0 +1,16 @@ +package com.exadel.frs.core.trainservice.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EmbeddingSimilarityResult { + + private String subject; + private float similarity; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingVerificationProcessResult.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingVerificationProcessResult.java new file mode 100644 index 0000000000..f29c0a10e2 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingVerificationProcessResult.java @@ -0,0 +1,16 @@ +package com.exadel.frs.core.trainservice.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EmbeddingVerificationProcessResult { + + private double[] embedding; + private float similarity; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsProcessResponse.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsProcessResponse.java new file mode 100644 index 0000000000..1d299bcc95 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsProcessResponse.java @@ -0,0 +1,5 @@ +package com.exadel.frs.core.trainservice.dto; + +public interface EmbeddingsProcessResponse { + +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsRecognitionProcessResponse.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsRecognitionProcessResponse.java new file mode 100644 index 0000000000..31629d3771 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsRecognitionProcessResponse.java @@ -0,0 +1,16 @@ +package com.exadel.frs.core.trainservice.dto; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EmbeddingsRecognitionProcessResponse implements EmbeddingsProcessResponse { + + private List result; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsRecognitionRequest.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsRecognitionRequest.java new file mode 100644 index 0000000000..8a94c2e386 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsRecognitionRequest.java @@ -0,0 +1,17 @@ +package com.exadel.frs.core.trainservice.dto; + +import javax.validation.constraints.NotEmpty; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EmbeddingsRecognitionRequest { + + @NotEmpty + private double[][] embeddings; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsVerificationProcessResponse.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsVerificationProcessResponse.java new file mode 100644 index 0000000000..99c990be35 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsVerificationProcessResponse.java @@ -0,0 +1,16 @@ +package com.exadel.frs.core.trainservice.dto; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EmbeddingsVerificationProcessResponse implements EmbeddingsProcessResponse { + + private List result; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsVerificationRequest.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsVerificationRequest.java new file mode 100644 index 0000000000..0185bdaf2f --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/EmbeddingsVerificationRequest.java @@ -0,0 +1,20 @@ +package com.exadel.frs.core.trainservice.dto; + +import javax.validation.constraints.NotEmpty; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EmbeddingsVerificationRequest { + + @NotEmpty + private double[] source; + + @NotEmpty + private double[][] targets; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FaceVerification.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FaceVerification.java index 7bf91ae072..d17281fccc 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FaceVerification.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FaceVerification.java @@ -16,22 +16,21 @@ package com.exadel.frs.core.trainservice.dto; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import static org.apache.commons.lang3.StringUtils.isEmpty; import com.exadel.frs.commonservice.dto.ExecutionTimeDto; +import com.exadel.frs.commonservice.sdk.faces.feign.dto.FacesAge; import com.exadel.frs.commonservice.sdk.faces.feign.dto.FacesBox; import com.exadel.frs.commonservice.sdk.faces.feign.dto.FacesGender; -import com.exadel.frs.commonservice.sdk.faces.feign.dto.FacesAge; import com.exadel.frs.commonservice.sdk.faces.feign.dto.FacesMask; +import com.exadel.frs.commonservice.sdk.faces.feign.dto.FacesPose; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.List; - -import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; @AllArgsConstructor @JsonInclude(NON_NULL) @@ -46,6 +45,7 @@ public class FaceVerification extends FaceProcessResponse { private List> landmarks; private FacesAge age; private FacesGender gender; + private FacesPose pose; private Double[] embedding; @JsonProperty(value = "execution_time") private ExecutionTimeDto executionTime; @@ -53,7 +53,7 @@ public class FaceVerification extends FaceProcessResponse { @Override public FaceVerification prepareResponse(ProcessImageParams processImageParams) { String facePlugins = processImageParams.getFacePlugins(); - if (StringUtils.isEmpty(facePlugins) || !facePlugins.contains(CALCULATOR)) { + if (isEmpty(facePlugins) || !facePlugins.contains(CALCULATOR)) { this.setEmbedding(null); } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesDetectionResponseDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesDetectionResponseDto.java index 6d2e6e3a92..dac9e7c084 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesDetectionResponseDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesDetectionResponseDto.java @@ -15,15 +15,17 @@ */ package com.exadel.frs.core.trainservice.dto; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import static org.apache.commons.lang3.StringUtils.isEmpty; import com.exadel.frs.commonservice.dto.FindFacesResultDto; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; -import org.springframework.util.StringUtils; - import java.util.List; - -import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; @Data @Builder @@ -44,7 +46,7 @@ public FacesDetectionResponseDto prepareResponse(ProcessImageParams processImage } String facePlugins = processImageParams.getFacePlugins(); - if (StringUtils.isEmpty(facePlugins) || !facePlugins.contains(CALCULATOR)) { + if (isEmpty(facePlugins) || !facePlugins.contains(CALCULATOR)) { this.getResult().forEach(r -> r.setEmbedding(null)); } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesMask.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesPose.java similarity index 57% rename from java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesMask.java rename to java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesPose.java index 9770c2377d..dac3fcc79d 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesMask.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesPose.java @@ -5,7 +5,9 @@ @Data @Accessors(chain = true) -public class FacesMask { - private Double probability; - private String value; +public class FacesPose { + + private Double pitch; + private Double roll; + private Double yaw; } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesRecognitionResponseDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesRecognitionResponseDto.java index 3af2095352..5165009dcd 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesRecognitionResponseDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FacesRecognitionResponseDto.java @@ -15,14 +15,16 @@ */ package com.exadel.frs.core.trainservice.dto; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import static org.apache.commons.lang3.StringUtils.isEmpty; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; -import org.springframework.util.StringUtils; - import java.util.List; - -import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; @Data @Builder @@ -43,7 +45,7 @@ public FacesRecognitionResponseDto prepareResponse(ProcessImageParams processIma } String facePlugins = processImageParams.getFacePlugins(); - if (StringUtils.isEmpty(facePlugins) || !facePlugins.contains(CALCULATOR)) { + if (isEmpty(facePlugins) || !facePlugins.contains(CALCULATOR)) { this.getResult().forEach(r -> r.setEmbedding(null)); } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FindFacesResultDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FindFacesResultDto.java index 07e2c4f8d9..b93f4c0e61 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FindFacesResultDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/FindFacesResultDto.java @@ -19,6 +19,7 @@ import com.exadel.frs.commonservice.dto.FacesAgeDto; import com.exadel.frs.commonservice.dto.FacesGenderDto; import com.exadel.frs.commonservice.dto.FacesMaskDto; +import com.exadel.frs.commonservice.dto.FacesPoseDto; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; @@ -37,6 +38,7 @@ public class FindFacesResultDto { private FacesAgeDto age; private FacesGenderDto gender; + private FacesPoseDto pose; private Double[] embedding; private FacesBox box; @JsonProperty(value = "execution_time") diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ModelValidationResult.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ModelValidationResult.java new file mode 100644 index 0000000000..9e814e88a4 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ModelValidationResult.java @@ -0,0 +1,18 @@ +package com.exadel.frs.core.trainservice.dto; + +import static com.exadel.frs.commonservice.enums.ValidationResult.FORBIDDEN; +import com.exadel.frs.commonservice.enums.ValidationResult; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class ModelValidationResult { + + private long modelId; + private ValidationResult result = FORBIDDEN; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/PluginsVersionsDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/PluginsVersionsDto.java index 6f34837153..dc4b80a988 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/PluginsVersionsDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/PluginsVersionsDto.java @@ -32,6 +32,8 @@ public class PluginsVersionsDto { private String age; private String gender; + private String pose; private String detector; private String calculator; + private String mask; } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ProcessEmbeddingsParams.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ProcessEmbeddingsParams.java new file mode 100644 index 0000000000..ae96f286b8 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ProcessEmbeddingsParams.java @@ -0,0 +1,14 @@ +package com.exadel.frs.core.trainservice.dto; + +import java.util.Map; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class ProcessEmbeddingsParams { + + private String apiKey; + private double[][] embeddings; + private Map additionalParams; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ProcessImageParams.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ProcessImageParams.java index ad0ac73bc2..35b20c33e5 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ProcessImageParams.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ProcessImageParams.java @@ -15,5 +15,6 @@ public class ProcessImageParams { private Double detProbThreshold; private String facePlugins; private Boolean status; + private Boolean detectFaces; private Map additionalParams; } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/SubjectDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/SubjectDto.java index 8e469c0234..f94dddc11c 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/SubjectDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/SubjectDto.java @@ -1,15 +1,16 @@ package com.exadel.frs.core.trainservice.dto; +import static com.exadel.frs.commonservice.system.global.RegExConstants.ALLOWED_SPECIAL_CHARACTERS; +import static com.exadel.frs.core.trainservice.system.global.Constants.SUBJECT_DESC; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiParam; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import javax.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import javax.validation.constraints.NotNull; - -import static com.exadel.frs.core.trainservice.system.global.Constants.SUBJECT_DESC; - @Data @NoArgsConstructor @AllArgsConstructor @@ -17,6 +18,8 @@ public class SubjectDto { @ApiParam(value = SUBJECT_DESC, required = true) @JsonProperty("subject") - @NotNull + @NotBlank(message = "Subject name cannot be empty") + @Size(min = 1, max = 50, message = "Subject name size must be between 1 and 50") + @Pattern(regexp = ALLOWED_SPECIAL_CHARACTERS, message = "The name cannot contain the following special characters: ';', '/', '\\'") private String subjectName; } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VerifyFacesResponse.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VerifyFacesResponse.java index 4717ee312b..aa7e249201 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VerifyFacesResponse.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VerifyFacesResponse.java @@ -16,15 +16,18 @@ package com.exadel.frs.core.trainservice.dto; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import static org.apache.commons.lang3.StringUtils.isEmpty; import com.exadel.frs.commonservice.dto.PluginsVersionsDto; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; -import org.springframework.util.StringUtils; - import java.util.List; - -import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; @Getter @Setter @@ -45,7 +48,7 @@ public class VerifyFacesResponse extends FaceProcessResponse { @Override public VerifyFacesResponse prepareResponse(ProcessImageParams processImageParams) { String facePlugins = processImageParams.getFacePlugins(); - if (StringUtils.isEmpty(facePlugins) || !facePlugins.contains(CALCULATOR)) { + if (isEmpty(facePlugins) || !facePlugins.contains(CALCULATOR)) { this.getProcessFileData().setEmbedding(null); this.faceMatches.forEach(fm -> fm.setEmbedding(null)); } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VerifyFacesResultDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VerifyFacesResultDto.java index 97fc5c21be..2ee418840d 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VerifyFacesResultDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VerifyFacesResultDto.java @@ -16,6 +16,7 @@ package com.exadel.frs.core.trainservice.dto; import com.exadel.frs.commonservice.dto.ExecutionTimeDto; +import com.exadel.frs.commonservice.sdk.faces.feign.dto.FacesMask; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; @@ -36,6 +37,7 @@ public class VerifyFacesResultDto { private FacesAge age; private FacesGender gender; + private FacesPose pose; private Double[] embedding; private FacesBox box; @JsonProperty(value = "execution_time") diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VersionConsistenceDto.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VersionConsistenceDto.java index 86d09dd9d7..02025df461 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VersionConsistenceDto.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/VersionConsistenceDto.java @@ -1,6 +1,7 @@ package com.exadel.frs.core.trainservice.dto; import com.fasterxml.jackson.annotation.JsonInclude; +import com.exadel.frs.commonservice.enums.AppStatus; import lombok.Builder; import lombok.Data; @@ -12,4 +13,5 @@ public class VersionConsistenceDto { Boolean demoFaceCollectionIsInconsistent; Boolean dbIsInconsistent; Boolean saveImagesToDB; + AppStatus status; } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/filter/SecurityValidationFilter.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/filter/SecurityValidationFilter.java index 6169bcf165..f49d954950 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/filter/SecurityValidationFilter.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/filter/SecurityValidationFilter.java @@ -16,14 +16,35 @@ package com.exadel.frs.core.trainservice.filter; +import static com.exadel.frs.commonservice.enums.ModelType.DETECTION; +import static com.exadel.frs.commonservice.enums.ModelType.RECOGNITION; +import static com.exadel.frs.commonservice.enums.ModelType.VERIFY; +import static com.exadel.frs.commonservice.enums.ValidationResult.OK; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.core.trainservice.system.global.Constants.X_FRS_API_KEY_HEADER; +import static java.util.Collections.emptyList; +import static java.util.Collections.list; +import static java.util.function.Function.identity; import com.exadel.frs.commonservice.enums.ModelType; -import com.exadel.frs.commonservice.enums.ValidationResult; import com.exadel.frs.commonservice.exception.BadFormatModelKeyException; import com.exadel.frs.commonservice.exception.IncorrectModelTypeException; import com.exadel.frs.commonservice.exception.ModelNotFoundException; import com.exadel.frs.commonservice.handler.ResponseExceptionHandler; +import com.exadel.frs.core.trainservice.cache.ModelStatisticCacheProvider; import com.exadel.frs.core.trainservice.service.ModelService; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.val; @@ -34,22 +55,6 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; - -import static com.exadel.frs.commonservice.enums.ModelType.*; -import static com.exadel.frs.commonservice.enums.ValidationResult.OK; -import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; -import static com.exadel.frs.core.trainservice.system.global.Constants.X_FRS_API_KEY_HEADER; -import static java.util.Collections.emptyList; -import static java.util.Collections.list; -import static java.util.function.Function.identity; - /** * Filter created to validate if this application has access to requested model */ @@ -65,6 +70,8 @@ public class SecurityValidationFilter implements Filter { private final ResponseExceptionHandler handler; private final ObjectMapper objectMapper; + private final ModelStatisticCacheProvider modelStatisticCacheProvider; + @Override public void init(final FilterConfig filterConfig) { @@ -80,7 +87,7 @@ public void doFilter( val httpResponse = (HttpServletResponse) servletResponse; String requestURI = httpRequest.getRequestURI(); - if (!requestURI.matches("^/(swagger|webjars|v2|api/v1/migrate|api/v1/consistence/status|api/v1/static).*$")) { + if (!requestURI.matches("^/(swagger|webjars|v2|api/v1/migrate|api/v1/consistence/status|api/v1/static|api/v1/config).*$")) { val headersMap = list(httpRequest.getHeaderNames()).stream() .collect(Collectors.>toMap( @@ -104,15 +111,18 @@ public void doFilter( return; } - ModelType modelType = getModelTypeByUrl(requestURI); - ValidationResult validationResult = modelService.validateModelKey(key, modelType); - if (validationResult != OK) { - String capitalize = ModelType.VERIFY.equals(modelType) ? VERIFICATION : StringUtils.capitalize(modelType.name().toLowerCase()); + val modelType = getModelTypeByUrl(requestURI); + val validationResult = modelService.validateModelKey(key, modelType); + if (validationResult.getResult() != OK) { + val capitalize = ModelType.VERIFY.equals(modelType) ? VERIFICATION : StringUtils.capitalize(modelType.name().toLowerCase()); val objectResponseEntity = handler.handleDefinedExceptions(new ModelNotFoundException(key, capitalize)); buildException(httpResponse, objectResponseEntity); return; } + if (requestURI.matches("^/(api/v1/recognition/recognize|api/v1/detection/detect|api/v1/verification/verify).*$")) { + modelStatisticCacheProvider.incrementRequestCount(validationResult.getModelId()); + } } else { val objectResponseEntity = handler.handleMissingRequestHeader(X_FRS_API_KEY_HEADER); buildException(httpResponse, objectResponseEntity); @@ -134,7 +144,8 @@ private void buildException(final HttpServletResponse response, final ResponseEn response.setStatus(responseEntity.getStatusCode().value()); response.setContentType(MediaType.APPLICATION_JSON_VALUE); response.getWriter().append(objectMapper.writeValueAsString(responseEntity.getBody())); - response.getWriter().flush(); + //response.getWriter().flush(); + //don't need to flush or close the writer } private ModelType getModelTypeByUrl(String url) { diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/mapper/EmbeddingMapper.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/mapper/EmbeddingMapper.java index d80cbd3824..7261ef01b3 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/mapper/EmbeddingMapper.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/mapper/EmbeddingMapper.java @@ -1,6 +1,6 @@ package com.exadel.frs.core.trainservice.mapper; -import com.exadel.frs.commonservice.entity.EmbeddingProjection; +import com.exadel.frs.commonservice.projection.EmbeddingProjection; import com.exadel.frs.core.trainservice.dto.EmbeddingDto; import org.mapstruct.Mapper; diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/mapper/FacesMapper.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/mapper/FacesMapper.java index d543dd4387..156b20fb25 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/mapper/FacesMapper.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/mapper/FacesMapper.java @@ -35,4 +35,4 @@ public interface FacesMapper { VerifyFacesResultDto toVerifyFacesResultDto(FindFacesResult facesResult); PluginsVersionsDto toPluginVersionsDto(PluginsVersions pluginsVersions); -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingService.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingService.java index d09fb46e46..4a64f7e88b 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingService.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingService.java @@ -1,12 +1,15 @@ package com.exadel.frs.core.trainservice.service; import com.exadel.frs.commonservice.entity.Embedding; -import com.exadel.frs.commonservice.entity.EmbeddingProjection; +import com.exadel.frs.commonservice.projection.EmbeddingProjection; +import com.exadel.frs.commonservice.projection.EnhancedEmbeddingProjection; import com.exadel.frs.commonservice.entity.Img; import com.exadel.frs.commonservice.repository.EmbeddingRepository; import com.exadel.frs.commonservice.repository.ImgRepository; import com.exadel.frs.core.trainservice.system.global.Constants; +import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +import lombok.val; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; @@ -16,7 +19,6 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Function; -import java.util.stream.Stream; @Service @RequiredArgsConstructor @@ -31,8 +33,8 @@ public int updateEmbedding(UUID embeddingId, double[] embedding, String calculat } @Transactional - public T doWithEmbeddingsStream(String apiKey, Function, T> func) { - try (Stream stream = embeddingRepository.findBySubjectApiKey(apiKey)) { + public T doWithEnhancedEmbeddingProjectionStream(String apiKey, Function, T> func) { + try (val stream = embeddingRepository.findBySubjectApiKey(apiKey)) { return func.apply(stream); } } @@ -50,8 +52,8 @@ public Optional getImg(String apiKey, UUID embeddingId) { return imgRepository.getImgByEmbeddingId(apiKey, embeddingId); } - public Page listEmbeddings(String apiKey, Pageable pageable) { - return embeddingRepository.findBySubjectApiKey(apiKey, pageable); + public Page listEmbeddings(String apiKey, String subjectName, Pageable pageable) { + return embeddingRepository.findBySubjectApiKeyAndSubjectName(apiKey, subjectName, pageable); } public boolean isDemoCollectionInconsistent() { diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsProcessService.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsProcessService.java new file mode 100644 index 0000000000..25ada19c35 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsProcessService.java @@ -0,0 +1,9 @@ +package com.exadel.frs.core.trainservice.service; + +import com.exadel.frs.core.trainservice.dto.EmbeddingsProcessResponse; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; + +public interface EmbeddingsProcessService extends FaceProcessService { + + EmbeddingsProcessResponse processEmbeddings(ProcessEmbeddingsParams processEmbeddingsParams); +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsRecognizeProcessServiceImpl.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsRecognizeProcessServiceImpl.java new file mode 100644 index 0000000000..fee61631d8 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsRecognizeProcessServiceImpl.java @@ -0,0 +1,64 @@ +package com.exadel.frs.core.trainservice.service; + +import static com.exadel.frs.core.trainservice.system.global.Constants.PREDICTION_COUNT; +import static java.math.RoundingMode.HALF_UP; +import com.exadel.frs.commonservice.exception.IncorrectPredictionCountException; +import com.exadel.frs.commonservice.sdk.faces.FacesApiClient; +import com.exadel.frs.core.trainservice.component.FaceClassifierPredictor; +import com.exadel.frs.core.trainservice.dto.EmbeddingRecognitionProcessResult; +import com.exadel.frs.core.trainservice.dto.EmbeddingSimilarityResult; +import com.exadel.frs.core.trainservice.dto.EmbeddingsRecognitionProcessResponse; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; +import com.exadel.frs.core.trainservice.mapper.FacesMapper; +import com.exadel.frs.core.trainservice.validation.ImageExtensionValidator; +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.stereotype.Service; + +@Service("recognitionService") +public class EmbeddingsRecognizeProcessServiceImpl extends FaceRecognizeProcessServiceImpl implements EmbeddingsProcessService { + + private final FaceClassifierPredictor classifierPredictor; + + public EmbeddingsRecognizeProcessServiceImpl(final FaceClassifierPredictor classifierPredictor, + final FacesApiClient facesApiClient, + final ImageExtensionValidator imageExtensionValidator, + final FacesMapper facesMapper) { + super(classifierPredictor, facesApiClient, imageExtensionValidator, facesMapper); + this.classifierPredictor = classifierPredictor; + } + + @Override + public EmbeddingsRecognitionProcessResponse processEmbeddings(final ProcessEmbeddingsParams processEmbeddingsParams) { + Integer predictionCount = (Integer) processEmbeddingsParams.getAdditionalParams().get(PREDICTION_COUNT); + if (predictionCount == null || (predictionCount == 0 || predictionCount < -1)) { + throw new IncorrectPredictionCountException(); + } + + String apiKey = processEmbeddingsParams.getApiKey(); + double[][] embeddings = processEmbeddingsParams.getEmbeddings(); + + List results = + Arrays.stream(embeddings) + .map(embedding -> processEmbedding(predictionCount, apiKey, embedding)) + .toList(); + + return new EmbeddingsRecognitionProcessResponse(results); + } + + private EmbeddingRecognitionProcessResult processEmbedding(final Integer predictionCount, final String apiKey, final double[] embedding) { + List> predictions = classifierPredictor.predict(apiKey, embedding, predictionCount); + List similarities = predictions.stream() + .map(this::processPrediction) + .toList(); + + return new EmbeddingRecognitionProcessResult(embedding, similarities); + } + + private EmbeddingSimilarityResult processPrediction(final Pair prediction) { + BigDecimal scaledPrediction = BigDecimal.valueOf(prediction.getLeft()).setScale(5, HALF_UP); + return new EmbeddingSimilarityResult(prediction.getRight(), scaledPrediction.floatValue()); + } +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsVerificationProcessServiceImpl.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsVerificationProcessServiceImpl.java new file mode 100644 index 0000000000..67530f0fa2 --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/EmbeddingsVerificationProcessServiceImpl.java @@ -0,0 +1,62 @@ +package com.exadel.frs.core.trainservice.service; + +import static java.math.RoundingMode.HALF_UP; +import com.exadel.frs.commonservice.exception.WrongEmbeddingCountException; +import com.exadel.frs.commonservice.sdk.faces.FacesApiClient; +import com.exadel.frs.core.trainservice.component.FaceClassifierPredictor; +import com.exadel.frs.core.trainservice.dto.EmbeddingVerificationProcessResult; +import com.exadel.frs.core.trainservice.dto.EmbeddingsVerificationProcessResponse; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; +import com.exadel.frs.core.trainservice.mapper.FacesMapper; +import com.exadel.frs.core.trainservice.validation.ImageExtensionValidator; +import java.math.BigDecimal; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.commons.lang3.ArrayUtils; +import org.springframework.stereotype.Service; + +@Service("verificationService") +public class EmbeddingsVerificationProcessServiceImpl extends FaceVerificationProcessServiceImpl implements EmbeddingsProcessService { + + private static final int MINIMUM_EMBEDDING_COUNT = 2; + + private final FaceClassifierPredictor classifierPredictor; + + public EmbeddingsVerificationProcessServiceImpl(final FaceClassifierPredictor classifierPredictor, + final FacesApiClient client, + final ImageExtensionValidator imageValidator, + final FacesMapper mapper) { + super(classifierPredictor, client, imageValidator, mapper); + this.classifierPredictor = classifierPredictor; + } + + @Override + public EmbeddingsVerificationProcessResponse processEmbeddings(final ProcessEmbeddingsParams processEmbeddingsParams) { + double[][] embeddings = processEmbeddingsParams.getEmbeddings(); + if (embeddings == null || (embeddings.length < MINIMUM_EMBEDDING_COUNT)) { + int embeddingCount = embeddings == null ? 0 : embeddings.length; + throw new WrongEmbeddingCountException(MINIMUM_EMBEDDING_COUNT, embeddingCount); + } + + double[] source = embeddings[0]; + double[][] targets = ArrayUtils.subarray(embeddings, 1, embeddings.length); + double[] similarities = classifierPredictor.verify(source, targets); + + List results = + IntStream.range(0, targets.length) + .mapToObj(i -> processEmbedding(targets[i], similarities[i])) + .sorted((e1, e2) -> Float.compare(e2.getSimilarity(), e1.getSimilarity())) + .collect(Collectors.toList()); + + return new EmbeddingsVerificationProcessResponse(results); + } + + private EmbeddingVerificationProcessResult processEmbedding(final double[] target, final double similarity) { + BigDecimal scaledSimilarity = BigDecimal.valueOf(similarity).setScale(5, HALF_UP); + return new EmbeddingVerificationProcessResult( + target, + scaledSimilarity.floatValue() + ); + } +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceDetectionProcessServiceImpl.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceDetectionProcessServiceImpl.java index 0c52a7e25d..d16b12f354 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceDetectionProcessServiceImpl.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceDetectionProcessServiceImpl.java @@ -28,10 +28,10 @@ public FacesDetectionResponseDto processImage(ProcessImageParams processImagePar if (processImageParams.getFile() != null) { MultipartFile file = (MultipartFile) processImageParams.getFile(); imageExtensionValidator.validate(file); - findFacesResponse = facesApiClient.findFaces(file, limit, detProbThreshold, facePlugins); + findFacesResponse = facesApiClient.findFaces(file, limit, detProbThreshold, facePlugins, true); } else { imageExtensionValidator.validateBase64(processImageParams.getImageBase64()); - findFacesResponse = facesApiClient.findFacesBase64(processImageParams.getImageBase64(), limit, detProbThreshold, facePlugins); + findFacesResponse = facesApiClient.findFacesBase64(processImageParams.getImageBase64(), limit, detProbThreshold, facePlugins,true); } FacesDetectionResponseDto facesDetectionResponseDto = facesMapper.toFacesDetectionResponseDto(findFacesResponse); diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceProcessService.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceProcessService.java index 09c609e2a4..c3c2424e74 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceProcessService.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceProcessService.java @@ -6,5 +6,4 @@ public interface FaceProcessService { FaceProcessResponse processImage(ProcessImageParams processImageParams); - } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceRecognizeProcessServiceImpl.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceRecognizeProcessServiceImpl.java index c0a45bca7c..831b76dcc3 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceRecognizeProcessServiceImpl.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceRecognizeProcessServiceImpl.java @@ -13,7 +13,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.math.BigDecimal; @@ -23,9 +22,8 @@ import static com.exadel.frs.core.trainservice.system.global.Constants.PREDICTION_COUNT; import static java.math.RoundingMode.HALF_UP; -@Service("recognitionService") -@RequiredArgsConstructor @Slf4j +@RequiredArgsConstructor public class FaceRecognizeProcessServiceImpl implements FaceProcessService { private final FaceClassifierPredictor classifierPredictor; @@ -46,10 +44,22 @@ public FacesRecognitionResponseDto processImage(ProcessImageParams processImageP if (processImageParams.getFile() != null) { MultipartFile file = (MultipartFile) processImageParams.getFile(); imageExtensionValidator.validate(file); - findFacesResponse = facesApiClient.findFacesWithCalculator(file, processImageParams.getLimit(), processImageParams.getDetProbThreshold(), processImageParams.getFacePlugins()); + findFacesResponse = facesApiClient.findFacesWithCalculator( + file, + processImageParams.getLimit(), + processImageParams.getDetProbThreshold(), + processImageParams.getFacePlugins(), + processImageParams.getDetectFaces() + ); } else { imageExtensionValidator.validateBase64(processImageParams.getImageBase64()); - findFacesResponse = facesApiClient.findFacesBase64WithCalculator(processImageParams.getImageBase64(), processImageParams.getLimit(), processImageParams.getDetProbThreshold(), processImageParams.getFacePlugins()); + findFacesResponse = facesApiClient.findFacesBase64WithCalculator( + processImageParams.getImageBase64(), + processImageParams.getLimit(), + processImageParams.getDetProbThreshold(), + processImageParams.getFacePlugins(), + processImageParams.getDetectFaces() + ); } val facesRecognitionDto = facesMapper.toFacesRecognitionResponseDto(findFacesResponse); diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceVerificationProcessServiceImpl.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceVerificationProcessServiceImpl.java index b1df251bd1..b92e0ecf75 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceVerificationProcessServiceImpl.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/FaceVerificationProcessServiceImpl.java @@ -14,7 +14,6 @@ import com.exadel.frs.core.trainservice.mapper.FacesMapper; import com.exadel.frs.core.trainservice.validation.ImageExtensionValidator; import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.springframework.web.multipart.MultipartFile; @@ -30,7 +29,6 @@ import static com.exadel.frs.core.trainservice.system.global.Constants.TARGET_IMAGE; import static java.math.RoundingMode.HALF_UP; -@Service("verificationService") @RequiredArgsConstructor public class FaceVerificationProcessServiceImpl implements FaceProcessService { @@ -93,7 +91,8 @@ FindFacesResponse findFace(MultipartFile photo, ProcessImageParams processImageP photo, processImageParams.getLimit(), processImageParams.getDetProbThreshold(), - processImageParams.getFacePlugins() + processImageParams.getFacePlugins(), + true ); } } @@ -115,7 +114,8 @@ FindFacesResponse findFace(String photo, ProcessImageParams processImageParams) photo, processImageParams.getLimit(), processImageParams.getDetProbThreshold(), - processImageParams.getFacePlugins() + processImageParams.getFacePlugins(), + true ); } } @@ -197,8 +197,10 @@ private FaceMatch getFaceMatch(FindFacesResult targetFacesResult, Double similar faceMatch.setEmbedding(verifyFacesResultDto.getEmbedding()); faceMatch.setAge(verifyFacesResultDto.getAge()); faceMatch.setGender(verifyFacesResultDto.getGender()); + faceMatch.setPose(verifyFacesResultDto.getPose()); faceMatch.setLandmarks(verifyFacesResultDto.getLandmarks()); faceMatch.setSimilarity(BigDecimal.valueOf(similarity).setScale(5, HALF_UP).floatValue()); + faceMatch.setMask(targetFacesResult.getMask()); return faceMatch; } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/ModelService.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/ModelService.java index 9db84a2cdf..13efe4acef 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/ModelService.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/ModelService.java @@ -16,11 +16,10 @@ package com.exadel.frs.core.trainservice.service; -import static com.exadel.frs.commonservice.enums.ValidationResult.FORBIDDEN; import static com.exadel.frs.commonservice.enums.ValidationResult.OK; import com.exadel.frs.commonservice.enums.ModelType; -import com.exadel.frs.commonservice.enums.ValidationResult; import com.exadel.frs.commonservice.repository.ModelRepository; +import com.exadel.frs.core.trainservice.dto.ModelValidationResult; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -32,7 +31,9 @@ public class ModelService { private final ModelRepository modelRepository; - public ValidationResult validateModelKey(final String modelKey, ModelType type) { - return modelRepository.findByApiKeyAndType(modelKey, type).isPresent() ? OK : FORBIDDEN; + public ModelValidationResult validateModelKey(final String modelKey, ModelType type) { + return modelRepository.findByApiKeyAndType(modelKey, type) + .map(model -> new ModelValidationResult(model.getId(), OK)) + .orElseGet(ModelValidationResult::new); } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/ModelStatisticService.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/ModelStatisticService.java new file mode 100644 index 0000000000..6ddd24b07f --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/ModelStatisticService.java @@ -0,0 +1,128 @@ +package com.exadel.frs.core.trainservice.service; + +import static com.exadel.frs.commonservice.enums.TableLockName.MODEL_STATISTIC_LOCK; +import static java.time.ZoneOffset.UTC; +import static java.time.ZonedDateTime.now; +import static java.time.temporal.ChronoUnit.HOURS; +import com.exadel.frs.commonservice.entity.ModelStatistic; +import com.exadel.frs.commonservice.repository.ModelRepository; +import com.exadel.frs.commonservice.repository.ModelStatisticRepository; +import com.exadel.frs.commonservice.repository.TableLockRepository; +import com.exadel.frs.core.trainservice.cache.ModelStatisticCacheProvider; +import com.exadel.frs.core.trainservice.util.CronExecution; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import javax.annotation.PostConstruct; +import javax.transaction.Transactional; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ModelStatisticService { + + private static final String CRON_EXPRESSION_PLACEHOLDER = "${statistic.model.cron-expression}"; + + @Value(CRON_EXPRESSION_PLACEHOLDER) + private String cronExpression; + private CronExecution cronExecution; + + private final ModelRepository modelRepository; + private final TableLockRepository lockRepository; + private final ModelStatisticRepository statisticRepository; + private final ModelStatisticCacheProvider statisticCacheProvider; + + @PostConstruct + private void postConstruct() { + cronExecution = new CronExecution(cronExpression); + } + + @Transactional + @Scheduled(cron = CRON_EXPRESSION_PLACEHOLDER, zone = "UTC") + public void updateAndRecordStatistics() { + if (statisticCacheProvider.isEmpty()) { + log.info("No statistic to update or record."); + return; + } + + val lastExecution = getLastExecution(); + + if (lastExecution == null) { + log.error("Couldn't update or record statistics due to can't calculate the execution time for your cron expression."); + statisticCacheProvider.invalidateCache(); + return; + } + + val cache = statisticCacheProvider.getCacheCopyAsMap(); + statisticCacheProvider.invalidateCache(); + + // Used to obtain a table lock. Only one application instance per time can execute the method. + lockRepository.lockByName(MODEL_STATISTIC_LOCK); + + val updatedStatistics = updateStatistics(cache, lastExecution); + val recordedStatistics = recordStatistics(cache, lastExecution); + + val updateCount = updatedStatistics.size(); + val recordCount = recordedStatistics.size(); + + val statistics = new ArrayList(updateCount + recordCount); + statistics.addAll(updatedStatistics); + statistics.addAll(recordedStatistics); + + statisticRepository.saveAll(statistics); + log.info("The statistics have been updated({}) and recorded({})", updateCount, recordCount); + } + + private List updateStatistics(final Map cache, final LocalDateTime createDate) { + val modelIds = cache.keySet(); + val statisticsToUpdate = statisticRepository.findAllByModelIdInAndCreatedDate(modelIds, createDate); + val updatedStatistics = new ArrayList(); + + statisticsToUpdate.forEach(statistic -> { + val cacheKey = statistic.getModel().getId(); + val cacheRequestCount = cache.get(cacheKey); + val totalRequestCount = statistic.getRequestCount() + cacheRequestCount; + + statistic.setRequestCount(totalRequestCount); + + updatedStatistics.add(statistic); + cache.remove(cacheKey); + }); + + return updatedStatistics; + } + + private List recordStatistics(final Map cache, final LocalDateTime createDate) { + val modelIds = cache.keySet(); + val models = modelRepository.findAllByIdIn(modelIds); + val recordedStatistics = new ArrayList(); + + models.forEach(model -> { + val cacheKey = model.getId(); + val cacheRequestCount = cache.get(cacheKey); + val statistic = ModelStatistic.builder() + .requestCount(cacheRequestCount) + .createdDate(createDate) + .model(model) + .build(); + + recordedStatistics.add(statistic); + }); + + return recordedStatistics; + } + + private LocalDateTime getLastExecution() { + return cronExecution.getLastExecutionBefore(now(UTC)) + .flatMap(current -> cronExecution.getLastExecutionBefore(current)) + .map(last -> last.toLocalDateTime().truncatedTo(HOURS)) + .orElse(null); + } +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/NotificationSenderService.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/NotificationSenderService.java index 8075bceac8..b0e509ef62 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/NotificationSenderService.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/NotificationSenderService.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.impossibl.postgres.api.jdbc.PGConnection; import com.impossibl.postgres.jdbc.PGDataSource; +import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; @@ -22,11 +23,18 @@ public class NotificationSenderService { private final PGDataSource pgNotificationDatasource; private PGConnection connection; + @PostConstruct + public void setUp() { + try { + this.connection = (PGConnection) pgNotificationDatasource.getConnection(); + } catch (SQLException e) { + log.error("Error during connection to Postgres", e); + } + } public void notifyCacheChange(CacheActionDto cacheActionDto) { try { - connection = (PGConnection) pgNotificationDatasource.getConnection(); - Statement statement = connection.createStatement(); + Statement statement = this.connection.createStatement(); try { ObjectMapper objectMapper = new ObjectMapper(); diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/SubjectService.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/SubjectService.java index bc534ee7b9..bd7b0dce91 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/service/SubjectService.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/service/SubjectService.java @@ -1,8 +1,11 @@ package com.exadel.frs.core.trainservice.service; +import static java.math.RoundingMode.HALF_UP; import com.exadel.frs.commonservice.entity.Embedding; import com.exadel.frs.commonservice.entity.Subject; +import com.exadel.frs.commonservice.exception.EmbeddingNotFoundException; import com.exadel.frs.commonservice.exception.TooManyFacesException; +import com.exadel.frs.commonservice.exception.WrongEmbeddingCountException; import com.exadel.frs.commonservice.sdk.faces.FacesApiClient; import com.exadel.frs.commonservice.sdk.faces.feign.dto.FindFacesResponse; import com.exadel.frs.commonservice.sdk.faces.feign.dto.FindFacesResult; @@ -12,29 +15,36 @@ import com.exadel.frs.core.trainservice.component.classifiers.EuclideanDistanceClassifier; import com.exadel.frs.core.trainservice.dao.SubjectDao; import com.exadel.frs.core.trainservice.dto.EmbeddingInfo; +import com.exadel.frs.core.trainservice.dto.EmbeddingVerificationProcessResult; +import com.exadel.frs.core.trainservice.dto.EmbeddingsVerificationProcessResponse; import com.exadel.frs.core.trainservice.dto.FaceVerification; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; import com.exadel.frs.core.trainservice.dto.ProcessImageParams; import com.exadel.frs.core.trainservice.system.global.Constants; +import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Service; - import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; -import java.math.BigDecimal; -import java.util.*; -import java.util.stream.Stream; - -import static java.math.RoundingMode.HALF_UP; - @Service @RequiredArgsConstructor @Slf4j public class SubjectService { + private static final int MINIMUM_EMBEDDING_COUNT = 1; private static final int MAX_FACES_TO_SAVE = 1; public static final int MAX_FACES_TO_RECOGNIZE = 2; @@ -79,11 +89,11 @@ public int removeAllSubjectEmbeddings(final String apiKey, final String subjectN return removed; } - public void deleteSubjectByName(final String apiKey, final String subjectName) { + public Pair deleteSubjectByName(final String apiKey, final String subjectName) { if (StringUtils.isBlank(subjectName)) { - deleteSubjectsByApiKey(apiKey); + return Pair.of(deleteSubjectsByApiKey(apiKey), null); } else { - deleteSubjectByNameAndApiKey(apiKey, subjectName); + return Pair.of(null, deleteSubjectByNameAndApiKey(apiKey, subjectName)); } } @@ -110,6 +120,17 @@ public Embedding removeSubjectEmbedding(final String apiKey, final UUID embeddin return embedding; } + public List removeSubjectEmbeddings(final String apiKey, final List embeddingIds){ + List result = new ArrayList<>(); + for (UUID id: embeddingIds) { + try { + result.add(removeSubjectEmbedding(apiKey, id)); + } catch (EmbeddingNotFoundException e){ + e.printStackTrace(); + } + } + return result; + } public boolean updateSubjectName(final String apiKey, final String oldSubjectName, final String newSubjectName) { if (StringUtils.isEmpty(newSubjectName) || newSubjectName.equals(oldSubjectName)) { @@ -139,7 +160,8 @@ public Pair saveCalculatedEmbedding( base64photo, MAX_FACES_TO_RECOGNIZE, detProbThreshold, - null + null, + true ); return saveCalculatedEmbedding( @@ -160,7 +182,8 @@ public Pair saveCalculatedEmbedding( file, MAX_FACES_TO_RECOGNIZE, detProbThreshold, - null + null, + true ); return saveCalculatedEmbedding( @@ -206,9 +229,11 @@ public Pair, PluginsVersions> verifyFace(ProcessImagePara FindFacesResponse findFacesResponse; if (processImageParams.getFile() != null) { MultipartFile file = (MultipartFile) processImageParams.getFile(); - findFacesResponse = facesApiClient.findFacesWithCalculator(file, processImageParams.getLimit(), processImageParams.getDetProbThreshold(), processImageParams.getFacePlugins()); + findFacesResponse = facesApiClient.findFacesWithCalculator(file, processImageParams.getLimit(), + processImageParams.getDetProbThreshold(), processImageParams.getFacePlugins(), true); } else { - findFacesResponse = facesApiClient.findFacesBase64WithCalculator(processImageParams.getImageBase64(), processImageParams.getLimit(), processImageParams.getDetProbThreshold(), processImageParams.getFacePlugins()); + findFacesResponse = facesApiClient.findFacesBase64WithCalculator(processImageParams.getImageBase64(), + processImageParams.getLimit(), processImageParams.getDetProbThreshold(), processImageParams.getFacePlugins(), true); } if (findFacesResponse == null) { @@ -247,6 +272,7 @@ public Pair, PluginsVersions> verifyFace(ProcessImagePara .embedding(findResult.getEmbedding()) .executionTime(findResult.getExecutionTime()) .age(findResult.getAge()) + .pose(findResult.getPose()) .mask(findResult.getMask()) .build() .prepareResponse(processImageParams); // do some tricks with obj @@ -259,4 +285,28 @@ public Pair, PluginsVersions> verifyFace(ProcessImagePara Boolean.TRUE.equals(processImageParams.getStatus()) ? findFacesResponse.getPluginsVersions() : null ); } + + public EmbeddingsVerificationProcessResponse verifyEmbedding(ProcessEmbeddingsParams processEmbeddingsParams) { + double[][] targets = processEmbeddingsParams.getEmbeddings(); + if (ArrayUtils.isEmpty(targets)) { + throw new WrongEmbeddingCountException(MINIMUM_EMBEDDING_COUNT, 0); + } + + UUID sourceId = (UUID) processEmbeddingsParams.getAdditionalParams().get(Constants.IMAGE_ID); + String apiKey = processEmbeddingsParams.getApiKey(); + + List results = + Arrays.stream(targets) + .map(target -> processTarget(target, sourceId, apiKey)) + .sorted((e1, e2) -> Float.compare(e2.getSimilarity(), e1.getSimilarity())) + .toList(); + + return new EmbeddingsVerificationProcessResponse(results); + } + + private EmbeddingVerificationProcessResult processTarget(double[] target, UUID sourceId, String apiKey) { + double similarity = predictor.verify(apiKey, target, sourceId); + float scaledSimilarity = BigDecimal.valueOf(similarity).setScale(5, HALF_UP).floatValue(); + return new EmbeddingVerificationProcessResult(target, scaledSimilarity); + } } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/system/global/Constants.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/system/global/Constants.java index b280402943..df0ad55d3b 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/system/global/Constants.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/system/global/Constants.java @@ -41,21 +41,27 @@ public class Constants { public static final String DET_PROB_THRESHOLD_DESC = "The minimal percent confidence that found face is actually a face."; public static final String FACE_PLUGINS_DESC = "Comma-separated types of face plugins. Empty value - face plugins disabled, returns only bounding boxes"; public static final String STATUS_DESC = "Special parameter to show execution_time and plugin_version fields. Empty value - both fields eliminated, true - both fields included"; + public static final String DETECT_FACES_DESC = "The parameter specifies whether to perform image detection or not"; public static final String PREDICTION_COUNT = "predictionCount"; public static final String STATUS_DEFAULT_VALUE = "false"; + public static final String DETECT_FACES_DEFAULT_VALUE = "true"; public static final String PREDICTION_COUNT_DEFAULT_VALUE = "1"; public static final String LIMIT_DEFAULT_VALUE = "0"; public static final String IMAGE_WITH_ONE_FACE_DESC = "A picture with one face (accepted formats: jpeg, png)."; public static final String IMAGE_ID_DESC = "Image Id from collection to compare with face."; + public static final String IMAGE_IDS_DESC = "List of image Ids from collection to compare with face"; public static final String SUBJECT_DESC = "Person's name to whom the face belongs to."; public static final String SUBJECT = "subject"; - public static final String SUBJECTS = "faces"; public static final String IMAGE_ID = "image_id"; public static final String SOURCE_IMAGE_DESC = "File to be verified"; public static final String TARGET_IMAGE_DESC = "Reference file to check the processed file"; public static final String STATUS = "status"; + public static final String DETECT_FACES = "detect_faces"; + public static final String SUBJECT_NAME_IS_EMPTY = "Subject name is empty"; + public static final String NUMBER_VALUE_EXAMPLE = "1"; public static final String DEMO_API_KEY = "00000000-0000-0000-0000-000000000002"; public static final String FACENET2018 = "Facenet2018"; public static final String SERVER_UUID = UUID.randomUUID().toString(); -} \ No newline at end of file + public static final String CACHE_CONTROL_HEADER_VALUE = "public, max-age=31536000"; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/system/swagger/SwaggerInfoProperties.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/system/swagger/SwaggerInfoProperties.java index 4356e01cfc..a2b8bda34c 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/system/swagger/SwaggerInfoProperties.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/system/swagger/SwaggerInfoProperties.java @@ -16,7 +16,7 @@ package com.exadel.frs.core.trainservice.system.swagger; -import static org.springframework.util.StringUtils.isEmpty; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; import lombok.Data; import lombok.val; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -46,24 +46,24 @@ public ApiInfo getApiInfo() { val builder = new ApiInfoBuilder(); - if (!isEmpty(this.contactName) - || !isEmpty(this.contactUrl) - || !isEmpty(this.contactEmail)) { + if (isNotEmpty(this.contactName) + || isNotEmpty(this.contactUrl) + || isNotEmpty(this.contactEmail)) { builder.contact(new Contact(this.contactName, this.contactUrl, this.contactEmail)); } - if (!isEmpty(this.description)) { + if (isNotEmpty(this.description)) { builder.description(this.description); } - if (!isEmpty(this.termsOfServiceUrl)) { + if (isNotEmpty(this.termsOfServiceUrl)) { builder.termsOfServiceUrl(this.termsOfServiceUrl); } - if (!isEmpty(this.title)) { + if (isNotEmpty(this.title)) { builder.title(this.title); } - if (!isEmpty(this.license)) { + if (isNotEmpty(this.license)) { builder.license(this.license); } - if (!isEmpty(this.version)) { + if (isNotEmpty(this.version)) { builder.version(this.version); } diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/util/ApplicationContextProvider.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/util/ApplicationContextProvider.java deleted file mode 100644 index 91e2461dd2..0000000000 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/util/ApplicationContextProvider.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.core.trainservice.util; - -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.stereotype.Component; - -@Component -public class ApplicationContextProvider implements ApplicationContextAware { - - private static ApplicationContext context; - - @Override - public void setApplicationContext(ApplicationContext ac) throws BeansException { - context = ac; - } - - public static ApplicationContext getApplicationContext() { - return context; - } - - public static T getBean(Class clazz) { - return getApplicationContext().getBean(clazz); - } -} \ No newline at end of file diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/util/CronExecution.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/util/CronExecution.java new file mode 100644 index 0000000000..08dec01cef --- /dev/null +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/util/CronExecution.java @@ -0,0 +1,25 @@ +package com.exadel.frs.core.trainservice.util; + +import static com.cronutils.model.CronType.SPRING; +import static com.cronutils.model.definition.CronDefinitionBuilder.instanceDefinitionFor; +import static com.cronutils.model.time.ExecutionTime.forCron; +import com.cronutils.model.definition.CronDefinition; +import com.cronutils.model.time.ExecutionTime; +import com.cronutils.parser.CronParser; +import java.time.ZonedDateTime; +import java.util.Optional; + +public class CronExecution { + + private final ExecutionTime executionTime; + + public CronExecution(final String cronExpression) { + CronDefinition cronDefinition = instanceDefinitionFor(SPRING); + CronParser cronParser = new CronParser(cronDefinition); + executionTime = forCron(cronParser.parse(cronExpression)); + } + + public Optional getLastExecutionBefore(final ZonedDateTime date) { + return executionTime.lastExecution(date); + } +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/util/MultipartFileData.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/util/MultipartFileData.java index 1c2def17c9..93052f1dad 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/util/MultipartFileData.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/util/MultipartFileData.java @@ -76,4 +76,4 @@ public void transferTo(File dest) throws IOException, IllegalStateException { fos.write(content); } } -} \ No newline at end of file +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/validation/ImageExtensionValidator.java b/java/api/src/main/java/com/exadel/frs/core/trainservice/validation/ImageExtensionValidator.java index 3339e646cb..35229fe122 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/validation/ImageExtensionValidator.java +++ b/java/api/src/main/java/com/exadel/frs/core/trainservice/validation/ImageExtensionValidator.java @@ -27,6 +27,7 @@ import org.springframework.web.multipart.MultipartFile; import static com.google.common.io.Files.getFileExtension; +import static org.apache.commons.lang3.StringUtils.isEmpty; @Component @RequiredArgsConstructor @@ -41,7 +42,7 @@ public void validate(final MultipartFile file) { val formats = imageProperties.getTypes(); String originalFilename = file.getOriginalFilename(); - val isWrongFormat = StringUtils.isEmpty(originalFilename) || !formats.contains(getFileExtension(originalFilename.toLowerCase())); + val isWrongFormat = isEmpty(originalFilename) || !formats.contains(getFileExtension(originalFilename.toLowerCase())); if (isWrongFormat) { throw new FileExtensionException(originalFilename); @@ -53,4 +54,4 @@ public void validateBase64(final String base64Image) { throw new InvalidBase64Exception(); } } -} \ No newline at end of file +} diff --git a/java/api/src/main/resources/application.yml b/java/api/src/main/resources/application.yml index a73cbfd292..91a77b73a5 100644 --- a/java/api/src/main/resources/application.yml +++ b/java/api/src/main/resources/application.yml @@ -1,5 +1,8 @@ server: - port: 8080 + port: ${API_PORT:8080} + tomcat: + relaxed-path-chars: [ '[', ']' ] + relaxed-query-chars: [ '[', ']' ] spring: liquibase: @@ -22,6 +25,7 @@ spring: url: ${POSTGRES_URL:jdbc:postgresql://compreface-postgres-db:5432/frs} username: ${POSTGRES_USER:postgres} password: ${POSTGRES_PASSWORD:postgres} + continue-on-error: true hikari: maximum-pool-size: 3 minimum-idle: 3 @@ -31,7 +35,12 @@ spring: properties: hibernate: default_schema: public - jdbc.lob.non_contextual_creation: true # fix for Caused by: java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgConnection.createClob() is not yet implemented. + jdbc: + lob.non_contextual_creation: true # fix for Caused by: java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgConnection.createClob() is not yet implemented. + batch_size: 10 + order_inserts: true + order_updates: true + batch_versioned_data: true format_sql: true dialect: org.hibernate.dialect.PostgreSQL10Dialect hibernate: @@ -42,8 +51,8 @@ spring: servlet: multipart: enabled: true - max-file-size: 10MB - max-request-size: 10MB + max-file-size: ${MAX_FILE_SIZE:5MB} + max-request-size: ${MAX_REQUEST_SIZE:10MB} # "environment" and "image" blocks should be same in those files: # * api/src/main/resources/application.properties @@ -67,11 +76,20 @@ image: - webp saveImagesToDB: ${SAVE_IMAGES_TO_DB:true} +statistic: + model: + cron-expression: ${MODEL_STATISTIC_CRON_EXPRESSION:0 0 * ? * *} + app: feign: appery-io: url: https://api.appery.io/rest/1/db/collections api-key: ${APPERY_API_KEY:#{null}} + faces: + connect-timeout: ${CONNECTION_TIMEOUT:10000} + read-timeout: ${READ_TIMEOUT:60000} + retryer: + max-attempts: ${MAX_ATTEMPTS:1} --- diff --git a/java/api/src/main/resources/logback-spring.xml b/java/api/src/main/resources/logback-spring.xml index 156c632014..ca9b871482 100644 --- a/java/api/src/main/resources/logback-spring.xml +++ b/java/api/src/main/resources/logback-spring.xml @@ -44,7 +44,7 @@ - + @@ -53,7 +53,7 @@ - + diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/DbHelper.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/DbHelper.java index e5ad438e79..588e371f5f 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/DbHelper.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/DbHelper.java @@ -1,22 +1,27 @@ package com.exadel.frs.core.trainservice; +import static com.exadel.frs.core.trainservice.ItemsBuilder.makeApp; +import static com.exadel.frs.core.trainservice.ItemsBuilder.makeEmbedding; +import static com.exadel.frs.core.trainservice.ItemsBuilder.makeImg; +import static com.exadel.frs.core.trainservice.ItemsBuilder.makeModel; +import static com.exadel.frs.core.trainservice.ItemsBuilder.makeSubject; import com.exadel.frs.commonservice.entity.Embedding; import com.exadel.frs.commonservice.entity.Img; import com.exadel.frs.commonservice.entity.Model; +import com.exadel.frs.commonservice.entity.ModelStatistic; import com.exadel.frs.commonservice.entity.Subject; import com.exadel.frs.commonservice.enums.ModelType; import com.exadel.frs.commonservice.repository.EmbeddingRepository; import com.exadel.frs.commonservice.repository.ImgRepository; import com.exadel.frs.commonservice.repository.ModelRepository; +import com.exadel.frs.commonservice.repository.ModelStatisticRepository; import com.exadel.frs.commonservice.repository.SubjectRepository; import com.exadel.frs.core.trainservice.repository.AppRepository; +import java.time.LocalDateTime; +import java.util.UUID; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.UUID; - -import static com.exadel.frs.core.trainservice.ItemsBuilder.*; - @Service public class DbHelper { @@ -29,6 +34,9 @@ public class DbHelper { @Autowired SubjectRepository subjectRepository; + @Autowired + ModelStatisticRepository modelStatisticRepository; + @Autowired EmbeddingRepository embeddingRepository; @@ -36,10 +44,13 @@ public class DbHelper { ImgRepository imgRepository; public Model insertModel() { - final String apiKey = UUID.randomUUID().toString(); + return insertModel(ModelType.RECOGNITION); + } + public Model insertModel(ModelType type) { + var apiKey = UUID.randomUUID().toString(); var app = appRepository.save(makeApp(apiKey)); - return modelRepository.save(makeModel(apiKey, ModelType.RECOGNITION, app)); + return modelRepository.save(makeModel(apiKey, type, app)); } public Subject insertSubject(Model model, String subjectName) { @@ -68,6 +79,15 @@ public Subject insertSubject(String subjectName) { return insertSubject(model.getApiKey(), subjectName); } + public ModelStatistic insertModelStatistic(Model model, int requestCount, final LocalDateTime createDate) { + var statistic = ModelStatistic.builder() + .createdDate(createDate) + .requestCount(requestCount) + .model(model) + .build(); + return modelStatisticRepository.save(statistic); + } + public Embedding insertEmbeddingNoImg(Subject subject) { return insertEmbeddingNoImg(subject, null); } diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/EmbeddedPostgreSQLTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/EmbeddedPostgreSQLTest.java index 793440133a..c04b71fd42 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/EmbeddedPostgreSQLTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/EmbeddedPostgreSQLTest.java @@ -1,7 +1,11 @@ package com.exadel.frs.core.trainservice; import com.exadel.frs.core.trainservice.config.IntegrationTest; +import com.exadel.frs.core.trainservice.service.NotificationReceiverService; +import com.exadel.frs.core.trainservice.service.NotificationSenderService; import io.zonky.test.db.AutoConfigureEmbeddedDatabase; +import javax.annotation.PostConstruct; +import javax.sql.DataSource; import liquibase.Contexts; import liquibase.LabelExpression; import liquibase.Liquibase; @@ -10,23 +14,33 @@ import liquibase.integration.spring.SpringResourceAccessor; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.env.Environment; import org.springframework.core.io.ResourceLoader; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; -import javax.annotation.PostConstruct; -import javax.sql.DataSource; - +@ActiveProfiles("test") @IntegrationTest @ExtendWith(SpringExtension.class) @AutoConfigureEmbeddedDatabase(beanName = "dsPg") public class EmbeddedPostgreSQLTest { + @MockBean + NotificationSenderService notificationSenderService; + + @MockBean + NotificationReceiverService notificationReceiverService; + @Autowired DataSource dataSource; @Autowired ResourceLoader resourceLoader; + @Autowired + private Environment env; + @PostConstruct public void initDatabase() { try { @@ -35,10 +49,23 @@ public void initDatabase() { new SpringResourceAccessor(resourceLoader), DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(dataSource.getConnection())) ); + setLiquibaseChangeLogParams(liquibase); liquibase.update(new Contexts(), new LabelExpression()); } catch (Exception e) { //manage exception e.printStackTrace(); } } + + private void setLiquibaseChangeLogParams(final Liquibase liquibase) { + String clientId = env.getProperty("spring.liquibase.parameters.common-client.client-id", "CommonClientId"); + String accessTokenValidity = env.getProperty("spring.liquibase.parameters.common-client.access-token-validity", "2400"); + String refreshTokenValidity = env.getProperty("spring.liquibase.parameters.common-client.refresh-token-validity", "1209600"); + String authorizedGrantTypes = env.getProperty("spring.liquibase.parameters.common-client.authorized-grant-types", "password,refresh_token"); + + liquibase.setChangeLogParameter("common-client.client-id", clientId); + liquibase.setChangeLogParameter("common-client.access-token-validity", accessTokenValidity); + liquibase.setChangeLogParameter("common-client.refresh-token-validity", refreshTokenValidity); + liquibase.setChangeLogParameter("common-client.authorized-grant-types", authorizedGrantTypes); + } } diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/ItemsBuilder.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/ItemsBuilder.java index f13b63ed7c..b14f9f9915 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/ItemsBuilder.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/ItemsBuilder.java @@ -2,6 +2,7 @@ import com.exadel.frs.commonservice.entity.*; import com.exadel.frs.commonservice.enums.ModelType; +import com.exadel.frs.commonservice.projection.EnhancedEmbeddingProjection; import com.exadel.frs.core.trainservice.system.global.Constants; import java.util.UUID; @@ -10,20 +11,20 @@ public class ItemsBuilder { public static App makeApp(String apiKey) { return App.builder() - .name("App" + System.currentTimeMillis()) - .guid(UUID.randomUUID().toString()) - .apiKey(apiKey) - .build(); + .name("App" + System.currentTimeMillis()) + .guid(UUID.randomUUID().toString()) + .apiKey(apiKey) + .build(); } public static Model makeModel(String apiKey, ModelType type, App app) { return Model.builder() - .apiKey(apiKey) - .name("Model" + UUID.randomUUID()) - .type(type) - .guid(UUID.randomUUID().toString()) - .app(app) - .build(); + .apiKey(apiKey) + .name("Model" + UUID.randomUUID()) + .type(type) + .guid(UUID.randomUUID().toString()) + .app(app) + .build(); } public static Embedding makeEmbedding(String subjectName, String apiKey) { @@ -33,6 +34,13 @@ public static Embedding makeEmbedding(String subjectName, String apiKey) { ); } + public static Embedding makeEmbedding(UUID embeddingId, String subjectName, String apiKey) { + return makeEmbedding( + makeSubject(apiKey, subjectName), + makeImg() + ).setId(embeddingId); + } + public static Embedding makeEmbedding(Subject subject, String calculator, double[] embedding, Img img) { return new Embedding() .setSubject(subject) @@ -49,6 +57,10 @@ public static Embedding makeEmbedding(Subject subject, Img img) { return makeEmbedding(subject, null, null, img); } + public static EnhancedEmbeddingProjection makeEnhancedEmbeddingProjection(String subject) { + return new EnhancedEmbeddingProjection(UUID.randomUUID(), new double[]{1.1, 2.2, 3.3}, subject); + } + public static Img makeImg(byte[] content) { return new Img() .setContent(content); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/EmbeddingCacheProviderTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/EmbeddingCacheProviderTest.java index bc304aa359..a2a0cc626f 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/EmbeddingCacheProviderTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/EmbeddingCacheProviderTest.java @@ -16,26 +16,25 @@ package com.exadel.frs.core.trainservice.cache; -import com.exadel.frs.commonservice.entity.Embedding; +import static com.exadel.frs.core.trainservice.ItemsBuilder.makeEnhancedEmbeddingProjection; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; +import com.exadel.frs.commonservice.projection.EnhancedEmbeddingProjection; import com.exadel.frs.core.trainservice.service.EmbeddingService; +import com.exadel.frs.core.trainservice.service.NotificationReceiverService; import com.exadel.frs.core.trainservice.service.NotificationSenderService; +import java.util.function.Function; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.function.Function; -import java.util.stream.Stream; - -import static com.exadel.frs.core.trainservice.ItemsBuilder.makeEmbedding; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; - @ExtendWith(MockitoExtension.class) class EmbeddingCacheProviderTest { @@ -47,28 +46,31 @@ class EmbeddingCacheProviderTest { @Mock private NotificationSenderService notificationSenderService; + @Mock + private NotificationReceiverService notificationReceiverService; + @InjectMocks private EmbeddingCacheProvider embeddingCacheProvider; @Test void getOrLoad() { - var embeddings = new Embedding[]{ - makeEmbedding("A", API_KEY), - makeEmbedding("B", API_KEY), - makeEmbedding("C", API_KEY) + var projections = new EnhancedEmbeddingProjection[]{ + makeEnhancedEmbeddingProjection("A"), + makeEnhancedEmbeddingProjection("B"), + makeEnhancedEmbeddingProjection("C") }; - when(embeddingService.doWithEmbeddingsStream(eq(API_KEY), any())) + when(embeddingService.doWithEnhancedEmbeddingProjectionStream(eq(API_KEY), any())) .thenAnswer(invocation -> { - var function = (Function, ?>) invocation.getArgument(1); - return function.apply(Stream.of(embeddings)); + var function = (Function, ?>) invocation.getArgument(1); + return function.apply(Stream.of(projections)); }); var actual = embeddingCacheProvider.getOrLoad(API_KEY); assertThat(actual, notNullValue()); assertThat(actual.getProjections(), notNullValue()); - assertThat(actual.getProjections().size(), is(embeddings.length)); + assertThat(actual.getProjections().size(), is(projections.length)); assertThat(actual.getEmbeddings(), notNullValue()); } } diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/EmbeddingCollectionTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/EmbeddingCollectionTest.java index daac66b3bf..0699ff7c4e 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/EmbeddingCollectionTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/EmbeddingCollectionTest.java @@ -16,15 +16,14 @@ package com.exadel.frs.core.trainservice.cache; -import com.exadel.frs.commonservice.entity.Embedding; -import com.exadel.frs.commonservice.entity.EmbeddingProjection; -import org.junit.jupiter.api.Test; - -import java.util.UUID; -import java.util.stream.Stream; - import static com.exadel.frs.core.trainservice.ItemsBuilder.makeEmbedding; +import static com.exadel.frs.core.trainservice.ItemsBuilder.makeEnhancedEmbeddingProjection; import static org.assertj.core.api.Assertions.assertThat; +import com.exadel.frs.commonservice.projection.EmbeddingProjection; +import com.exadel.frs.commonservice.projection.EnhancedEmbeddingProjection; +import java.util.UUID; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; class EmbeddingCollectionTest { @@ -53,50 +52,50 @@ void testAddToEmpty() { @Test void testCreate() { - var embedding1 = makeEmbedding("A", API_KEY); - var embedding2 = makeEmbedding("B", API_KEY); - var embedding3 = makeEmbedding("C", API_KEY); - var embeddings = new Embedding[]{embedding1, embedding2, embedding3}; - var embeddingCollection = EmbeddingCollection.from(Stream.of(embeddings)); + var projection1 = makeEnhancedEmbeddingProjection("A"); + var projection2 = makeEnhancedEmbeddingProjection("B"); + var projection3 = makeEnhancedEmbeddingProjection("C"); + var projections = new EnhancedEmbeddingProjection[]{projection1, projection2, projection3}; + var embeddingCollection = EmbeddingCollection.from(Stream.of(projections)); assertThat(embeddingCollection).isNotNull(); assertThat(embeddingCollection.getIndexMap()).isNotNull(); - assertThat(embeddingCollection.getIndexMap()).hasSize(embeddings.length); + assertThat(embeddingCollection.getIndexMap()).hasSize(projections.length); - assertThat(embeddingCollection.getIndexMap()).containsEntry(0, EmbeddingProjection.from(embedding1)); - assertThat(embeddingCollection.getIndexMap()).containsEntry(1, EmbeddingProjection.from(embedding2)); - assertThat(embeddingCollection.getIndexMap()).containsEntry(2, EmbeddingProjection.from(embedding3)); + assertThat(embeddingCollection.getIndexMap()).containsEntry(0, EmbeddingProjection.from(projection1)); + assertThat(embeddingCollection.getIndexMap()).containsEntry(1, EmbeddingProjection.from(projection2)); + assertThat(embeddingCollection.getIndexMap()).containsEntry(2, EmbeddingProjection.from(projection3)); } @Test void testAdd() { - var embeddings = new Embedding[]{ - makeEmbedding("A", API_KEY), - makeEmbedding("B", API_KEY), - makeEmbedding("C", API_KEY) + var projections = new EnhancedEmbeddingProjection[]{ + makeEnhancedEmbeddingProjection("A"), + makeEnhancedEmbeddingProjection("B"), + makeEnhancedEmbeddingProjection("C") }; - var embeddingCollection = EmbeddingCollection.from(Stream.of(embeddings)); + var embeddingCollection = EmbeddingCollection.from(Stream.of(projections)); var newEmbedding = makeEmbedding("D", API_KEY); var key = embeddingCollection.addEmbedding(newEmbedding); assertThat(key).isNotNull(); - assertThat(embeddingCollection.getProjections()).hasSize(embeddings.length + 1); - assertThat(embeddingCollection.getIndexMap()).containsEntry(embeddings.length, EmbeddingProjection.from(newEmbedding)); + assertThat(embeddingCollection.getProjections()).hasSize(projections.length + 1); + assertThat(embeddingCollection.getIndexMap()).containsEntry(projections.length, EmbeddingProjection.from(newEmbedding)); } @Test void testRemove() { - var embedding1 = makeEmbedding("A", API_KEY); - var embedding2 = makeEmbedding("B", API_KEY); - var embedding3 = makeEmbedding("C", API_KEY); - var embeddings = new Embedding[]{embedding1, embedding2, embedding3}; - var embeddingCollection = EmbeddingCollection.from(Stream.of(embeddings)); + var projection1 = makeEnhancedEmbeddingProjection("A"); + var projection2 = makeEnhancedEmbeddingProjection("B"); + var projection3 = makeEnhancedEmbeddingProjection("C"); + var projections = new EnhancedEmbeddingProjection[]{projection1, projection2, projection3}; + var embeddingCollection = EmbeddingCollection.from(Stream.of(projections)); - embeddingCollection.removeEmbedding(embedding1); + embeddingCollection.removeEmbedding(EmbeddingProjection.from(projection1)); - assertThat(embeddingCollection.getProjections()).hasSize(embeddings.length - 1); - assertThat(embeddingCollection.getIndexMap()).containsEntry(0, EmbeddingProjection.from(embedding2)); - assertThat(embeddingCollection.getIndexMap()).containsEntry(1, EmbeddingProjection.from(embedding3)); + assertThat(embeddingCollection.getProjections()).hasSize(projections.length - 1); + assertThat(embeddingCollection.getIndexMap()).containsEntry(0, EmbeddingProjection.from(projection2)); + assertThat(embeddingCollection.getIndexMap()).containsEntry(1, EmbeddingProjection.from(projection3)); } } \ No newline at end of file diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/ModelStatisticCacheProviderTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/ModelStatisticCacheProviderTest.java new file mode 100644 index 0000000000..0047726132 --- /dev/null +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/cache/ModelStatisticCacheProviderTest.java @@ -0,0 +1,54 @@ +package com.exadel.frs.core.trainservice.cache; + +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ModelStatisticCacheProviderTest { + + private final ModelStatisticCacheProvider cacheProvider = new ModelStatisticCacheProvider(); + + @AfterEach + void cleanup() { + cacheProvider.invalidateCache(); + } + + @Test + void shouldIncrementCacheValueWhenIncrementRequestCountMethodInvoked() { + cacheProvider.incrementRequestCount(1L); + + cacheProvider.incrementRequestCount(2L); + cacheProvider.incrementRequestCount(2L); + + cacheProvider.incrementRequestCount(3L); + cacheProvider.incrementRequestCount(3L); + cacheProvider.incrementRequestCount(3L); + + var cache = cacheProvider.getCacheCopyAsMap(); + + assertThat(cache).isNotNull().hasSize(3) + .containsEntry(1L, 1) + .containsEntry(2L, 2) + .containsEntry(3L, 3); + } + + @Test + void shouldInvalidateAllCacheWhenInvalidateCacheMethodInvoked() { + cacheProvider.incrementRequestCount(1L); + cacheProvider.incrementRequestCount(2L); + cacheProvider.incrementRequestCount(3L); + + var cacheBefore = cacheProvider.getCacheCopyAsMap(); + + assertThat(cacheBefore).isNotNull().hasSize(3); + + cacheProvider.invalidateCache(); + + var cacheAfter = cacheProvider.getCacheCopyAsMap(); + + assertThat(cacheAfter).isNotNull().isEmpty(); + } +} diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/component/migration/MigrationComponentTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/component/migration/MigrationComponentTest.java index 1c7ef348db..4c89494eb4 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/component/migration/MigrationComponentTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/component/migration/MigrationComponentTest.java @@ -42,7 +42,7 @@ void testRecalculateEmbeddingsWithOutdatedCalculator() { when(feignClient.getStatus()) .thenReturn(new FacesStatusResponse().setCalculatorVersion(currentCalculator)); - when(feignClient.findFaces(any(), any(), any(), any())) + when(feignClient.findFaces(any(), any(), any(), any(), any())) .thenReturn(FindFacesResponse.builder() .result(List.of(FindFacesResult.builder().embedding(newEmbeddingArray).build())) .build()); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/DetectionControllerTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/DetectionControllerTest.java index df48f69022..6be9e84735 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/DetectionControllerTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/DetectionControllerTest.java @@ -102,7 +102,7 @@ void testDetectFacesException(BasicException exception, ResultMatcher matcher) t String fileName = "file"; val mockFile = new MockMultipartFile(fileName, "test data".getBytes()); doNothing().when(validator).validate(mockFile); - when(client.findFaces(any(), any(), any(), any())).thenThrow(exception); + when(client.findFaces(any(), any(), any(), any(), any())).thenThrow(exception); // when mockMvc.perform( @@ -122,7 +122,7 @@ void testDetect() throws Exception { val mockFile = new MockMultipartFile(fileName, "test data".getBytes()); val findResponse = new FindFacesResponse(); doNothing().when(validator).validate(mockFile); - when(client.findFaces(any(), any(), any(), any())).thenReturn(findResponse); + when(client.findFaces(any(), any(), any(), any(), any())).thenReturn(findResponse); // when mockMvc.perform( @@ -139,7 +139,7 @@ void testDetectBase64() throws Exception { // given val findResponse = new FindFacesResponse(); doNothing().when(validator).validateBase64(any()); - when(client.findFacesBase64(any(), any(), any(), any())).thenReturn(findResponse); + when(client.findFacesBase64(any(), any(), any(), any(), any())).thenReturn(findResponse); Base64File request = new Base64File(); request.setContent(Base64.getEncoder().encodeToString(new byte[]{(byte) 0xCA})); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/EmbeddingControllerTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/EmbeddingControllerTest.java index 67bee25903..fc01fde1cd 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/EmbeddingControllerTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/EmbeddingControllerTest.java @@ -17,7 +17,7 @@ package com.exadel.frs.core.trainservice.controller; import com.exadel.frs.commonservice.entity.Embedding; -import com.exadel.frs.commonservice.entity.EmbeddingProjection; +import com.exadel.frs.commonservice.projection.EmbeddingProjection; import com.exadel.frs.commonservice.entity.Img; import com.exadel.frs.commonservice.entity.Subject; import com.exadel.frs.commonservice.exception.EmbeddingNotFoundException; @@ -180,7 +180,7 @@ void testDownloadImgNotFound() throws Exception { @Test void testListEmbeddings() throws Exception { - when(embeddingService.listEmbeddings(eq(API_KEY), any())) + when(embeddingService.listEmbeddings(eq(API_KEY), eq(null), any())) .thenReturn(new PageImpl<>( List.of( new EmbeddingProjection(UUID.randomUUID(), "name1"), @@ -201,6 +201,28 @@ void testListEmbeddings() throws Exception { .andExpect(jsonPath("$.total_elements", is(12))); } + @Test + void testListEmbeddingsWithSubjectName() throws Exception { + var subjectName = "Johnny Depp"; + when(embeddingService.listEmbeddings(eq(API_KEY), eq(subjectName), any())) + .thenReturn(new PageImpl<>( + List.of(new EmbeddingProjection(UUID.randomUUID(), subjectName)), + PageRequest.of(1, 10), // second page + 12 + )); + + mockMvc.perform( + get(API_V1 + "/recognition/faces") + .queryParam("subject", subjectName) + .header(X_FRS_API_KEY_HEADER, API_KEY) + ).andExpect(status().isOk()) + .andExpect(jsonPath("$.faces.length()", is(1))) + .andExpect(jsonPath("$.page_number", is(1))) // page number + .andExpect(jsonPath("$.page_size", is(10))) // page size + .andExpect(jsonPath("$.total_pages", is(2))) + .andExpect(jsonPath("$.total_elements", is(11))); + } + @Test void testListEmbeddingsFail() throws Exception { var expectedContent = "{\"message\":\"" + String.format("Missing header: %s", X_FRS_API_KEY_HEADER) + "\",\"code\":20}"; diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/MigrateControllerTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/MigrateControllerTest.java index 12ea0abba7..aa5eed5363 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/MigrateControllerTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/MigrateControllerTest.java @@ -21,6 +21,7 @@ import com.exadel.frs.core.trainservice.component.migration.MigrationComponent; import com.exadel.frs.core.trainservice.component.migration.MigrationStatusStorage; import com.exadel.frs.core.trainservice.config.IntegrationTest; +import com.exadel.frs.core.trainservice.dto.ModelValidationResult; import com.exadel.frs.core.trainservice.service.ModelService; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -28,6 +29,7 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.web.servlet.MockMvc; +import static com.exadel.frs.commonservice.enums.ValidationResult.OK; import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; @@ -52,7 +54,9 @@ class MigrateControllerTest extends EmbeddedPostgreSQLTest { @Test void migrate() throws Exception { - when(modelService.validateModelKey(anyString(), any())).thenReturn(ValidationResult.OK); + var validationResult = new ModelValidationResult(1L, OK); + + when(modelService.validateModelKey(anyString(), any())).thenReturn(validationResult); mockMvc.perform(post(API_V1 + "/migrate")) .andExpect(status().isOk()) diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/RecognizeControllerTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/RecognizeControllerTest.java index 69f1a4baed..de16c02669 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/RecognizeControllerTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/RecognizeControllerTest.java @@ -21,6 +21,7 @@ import com.exadel.frs.commonservice.sdk.faces.feign.dto.FindFacesResponse; import com.exadel.frs.commonservice.sdk.faces.feign.dto.FindFacesResult; import com.exadel.frs.commonservice.system.global.Constants; +import com.exadel.frs.core.trainservice.EmbeddedPostgreSQLTest; import com.exadel.frs.core.trainservice.component.FaceClassifierPredictor; import com.exadel.frs.core.trainservice.config.IntegrationTest; import com.exadel.frs.core.trainservice.dto.Base64File; @@ -53,7 +54,7 @@ @IntegrationTest @AutoConfigureMockMvc -class RecognizeControllerTest { +class RecognizeControllerTest extends EmbeddedPostgreSQLTest { @Autowired private MockMvc mockMvc; @@ -64,11 +65,6 @@ class RecognizeControllerTest { @MockBean private ImageExtensionValidator validator; - @Mock - private NotificationSenderService notificationSenderService; - @MockBean - private NotificationReceiverService notificationReceiverService; - @MockBean private FacesApiClient client; @@ -89,7 +85,7 @@ void recognize() throws Exception { )) .build(); - when(client.findFacesWithCalculator(any(), any(), any(), isNull())).thenReturn(findFacesResponse); + when(client.findFacesWithCalculator(any(), any(), any(), isNull(), any())).thenReturn(findFacesResponse); when(predictor.predict(any(), any(), anyInt())).thenReturn(List.of(Pair.of(1.0, ""))); doNothing().when(validator).validate(mockFile); @@ -110,7 +106,7 @@ void recognizeBase64() throws Exception { )) .build(); - when(client.findFacesBase64WithCalculator(any(), any(), any(), isNull())).thenReturn(findFacesResponse); + when(client.findFacesBase64WithCalculator(any(), any(), any(), isNull(), any())).thenReturn(findFacesResponse); when(predictor.predict(any(), any(), anyInt())).thenReturn(List.of(Pair.of(1.0, ""))); doNothing().when(validator).validateBase64(any()); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/VerifyControllerTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/VerifyControllerTest.java index fb65343b7c..0b6674456f 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/VerifyControllerTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/controller/VerifyControllerTest.java @@ -64,7 +64,7 @@ void verifyFaces() throws Exception { )) .build(); - when(client.findFacesWithCalculator(any(), any(), any(), isNull())).thenReturn(findFacesResponse); + when(client.findFacesWithCalculator(any(), any(), any(), isNull(), any())).thenReturn(findFacesResponse); when(predictor.verify(any(), any())).thenReturn(new double[]{100d}); val firstFile = new MockMultipartFile("source_image", "test data".getBytes()); @@ -78,7 +78,7 @@ void verifyFaces() throws Exception { ).andExpect(status().isOk()); verify(validator, times(2)).validate(any()); - verify(client, times(2)).findFacesWithCalculator(any(), any(), any(), isNull()); + verify(client, times(2)).findFacesWithCalculator(any(), any(), any(), isNull(), any()); verify(predictor).verify(any(), any(double[][].class)); verifyNoMoreInteractions(validator, client, predictor); } @@ -93,7 +93,7 @@ void verifyFacesBase64() throws Exception { )) .build(); - when(client.findFacesBase64WithCalculator(any(), any(), any(), anyString())).thenReturn(findFacesResponse); + when(client.findFacesBase64WithCalculator(any(), any(), any(), anyString(), any())).thenReturn(findFacesResponse); when(predictor.verify(any(), any())).thenReturn(new double[]{100d}); VerifySourceTargetRequest request = new VerifySourceTargetRequest(); @@ -110,7 +110,7 @@ void verifyFacesBase64() throws Exception { ).andExpect(status().isOk()); verify(validator, times(2)).validateBase64(any()); - verify(client, times(2)).findFacesBase64WithCalculator(any(), any(), any(), anyString()); + verify(client, times(2)).findFacesBase64WithCalculator(any(), any(), any(), anyString(), any()); verify(predictor).verify(any(), any(double[][].class)); verifyNoMoreInteractions(validator, client, predictor); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/dao/SubjectDaoTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/dao/SubjectDaoTest.java index fea7f63627..6c96abea49 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/dao/SubjectDaoTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/dao/SubjectDaoTest.java @@ -10,6 +10,7 @@ import com.exadel.frs.core.trainservice.DbHelper; import com.exadel.frs.core.trainservice.EmbeddedPostgreSQLTest; import com.exadel.frs.core.trainservice.dto.EmbeddingInfo; +import com.exadel.frs.core.trainservice.service.NotificationReceiverService; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -17,6 +18,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Pageable; import java.util.Optional; @@ -104,8 +106,8 @@ void testRemoveAllSubjectEmbeddings() { assertThat(embeddingRepository.findBySubjectId(subject.getId())).isEmpty(); // no images assertThat(imgRepository.getImgByEmbeddingId(subject.getApiKey(), embedding.getId())).isEmpty(); - // subject still exists - assertThat(subjectRepository.findById(subject.getId())).isPresent(); + // the subject doesn't exist anymore + assertThat(subjectRepository.findById(subject.getId())).isEmpty(); } @Test diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/filter/SecurityValidationFilterTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/filter/SecurityValidationFilterTest.java index 1cfce02737..9ab763255e 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/filter/SecurityValidationFilterTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/filter/SecurityValidationFilterTest.java @@ -16,21 +16,28 @@ package com.exadel.frs.core.trainservice.filter; -import static com.exadel.frs.core.trainservice.system.global.Constants.*; +import static com.exadel.frs.commonservice.enums.ValidationResult.FORBIDDEN; +import static com.exadel.frs.commonservice.enums.ValidationResult.OK; +import static com.exadel.frs.core.trainservice.system.global.Constants.API_V1; +import static com.exadel.frs.core.trainservice.system.global.Constants.RECOGNIZE; +import static com.exadel.frs.core.trainservice.system.global.Constants.X_FRS_API_KEY_HEADER; import static java.util.Collections.emptyEnumeration; import static java.util.Collections.enumeration; import static java.util.Collections.singletonList; -import static org.mockito.ArgumentMatchers.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; - +import com.exadel.frs.commonservice.enums.ModelType; import com.exadel.frs.commonservice.exception.BadFormatModelKeyException; import com.exadel.frs.commonservice.exception.ModelNotFoundException; import com.exadel.frs.commonservice.handler.ResponseExceptionHandler; -import com.exadel.frs.commonservice.enums.ModelType; -import com.exadel.frs.commonservice.enums.ValidationResult; +import com.exadel.frs.core.trainservice.cache.ModelStatisticCacheProvider; +import com.exadel.frs.core.trainservice.dto.ModelValidationResult; import com.exadel.frs.core.trainservice.service.ModelService; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -47,7 +54,7 @@ import org.springframework.http.MediaType; import org.springframework.util.StringUtils; -public class SecurityValidationFilterTest { +class SecurityValidationFilterTest { private static final String SHORT_API_KEY = "9892f9e2-1844-46f3-a710-72e"; private static final String VALID_API_KEY = "11f4cb4b-ea5a-45d4-8da7-863fea07c40a"; @@ -62,6 +69,9 @@ public class SecurityValidationFilterTest { @Mock private ModelService modelService; + @Mock + private ModelStatisticCacheProvider modelStatisticCacheProvider; + @InjectMocks private SecurityValidationFilter securityValidationFilter; @@ -78,11 +88,11 @@ void setUp() throws IOException { filterChain = mock(FilterChain.class); when(httpServletResponse.getWriter()).thenReturn(new PrintWriter(new StringWriter())); - when(httpServletRequest.getRequestURI()).thenReturn(API_V1+ RECOGNIZE); + when(httpServletRequest.getRequestURI()).thenReturn(API_V1 + RECOGNIZE); } @Test - public void testDoFilterWithShortApiKey() throws IOException, ServletException { + void testDoFilterWithShortApiKey() throws IOException, ServletException { when(httpServletRequest.getHeaderNames()).thenReturn(enumeration(singletonList(X_FRS_API_KEY_HEADER))); when(httpServletRequest.getHeaders(X_FRS_API_KEY_HEADER)).thenReturn(enumeration(singletonList(SHORT_API_KEY))); when(exceptionHandler.handleDefinedExceptions(any())).thenCallRealMethod(); @@ -97,7 +107,7 @@ public void testDoFilterWithShortApiKey() throws IOException, ServletException { } @Test - public void testDoFilterWithoutApiKey() throws IOException, ServletException { + void testDoFilterWithoutApiKey() throws IOException, ServletException { when(httpServletRequest.getHeaderNames()).thenReturn(emptyEnumeration()); when(httpServletRequest.getHeaders(X_FRS_API_KEY_HEADER)).thenReturn(emptyEnumeration()); when(exceptionHandler.handleMissingRequestHeader(anyString())).thenCallRealMethod(); @@ -112,10 +122,12 @@ public void testDoFilterWithoutApiKey() throws IOException, ServletException { } @Test - public void testDoFilterWithValidApiKey() throws IOException, ServletException { + void testDoFilterWithValidApiKey() throws IOException, ServletException { + var validationResult = new ModelValidationResult(1L, OK); + when(httpServletRequest.getHeaderNames()).thenReturn(enumeration(singletonList(X_FRS_API_KEY_HEADER))); when(httpServletRequest.getHeaders(X_FRS_API_KEY_HEADER)).thenReturn(enumeration(singletonList(VALID_API_KEY))); - when(modelService.validateModelKey(anyString(), any(ModelType.class))).thenReturn(ValidationResult.OK); + when(modelService.validateModelKey(anyString(), any(ModelType.class))).thenReturn(validationResult); securityValidationFilter.doFilter(httpServletRequest, httpServletResponse, filterChain); @@ -123,23 +135,28 @@ public void testDoFilterWithValidApiKey() throws IOException, ServletException { } @Test - public void testDoFilterWithNonExistentApiKey() throws IOException, ServletException { + void testDoFilterWithNonExistentApiKey() throws IOException, ServletException { + var validationResult = new ModelValidationResult(1L, FORBIDDEN); + when(httpServletRequest.getHeaderNames()).thenReturn(enumeration(singletonList(X_FRS_API_KEY_HEADER))); when(httpServletRequest.getHeaders(X_FRS_API_KEY_HEADER)).thenReturn(enumeration(singletonList(VALID_API_KEY))); - when(modelService.validateModelKey(anyString(), eq(ModelType.RECOGNITION))).thenReturn(ValidationResult.FORBIDDEN); + when(modelService.validateModelKey(anyString(), eq(ModelType.RECOGNITION))).thenReturn(validationResult); when(exceptionHandler.handleDefinedExceptions(any())).thenCallRealMethod(); securityValidationFilter.doFilter(httpServletRequest, httpServletResponse, filterChain); verify(httpServletResponse).setStatus( - exceptionHandler.handleDefinedExceptions(new ModelNotFoundException(VALID_API_KEY, StringUtils.capitalize(ModelType.RECOGNITION.name().toLowerCase()))) + exceptionHandler.handleDefinedExceptions(new ModelNotFoundException( + VALID_API_KEY, + StringUtils.capitalize(ModelType.RECOGNITION.name().toLowerCase()) + )) .getStatusCode() .value() ); } @Test - public void testDoFilterWithNotValidApiKey() throws IOException, ServletException { + void testDoFilterWithNotValidApiKey() throws IOException, ServletException { when(httpServletRequest.getHeaderNames()).thenReturn(enumeration(singletonList(X_FRS_API_KEY_HEADER))); when(httpServletRequest.getHeaders(X_FRS_API_KEY_HEADER)).thenReturn(enumeration(singletonList(NOT_VALID_API_KEY))); when(exceptionHandler.handleDefinedExceptions(any())).thenCallRealMethod(); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/repository/EmbeddingRepositoryTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/repository/EmbeddingRepositoryTest.java index 2a18e04ba2..3b02eb8947 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/repository/EmbeddingRepositoryTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/repository/EmbeddingRepositoryTest.java @@ -130,8 +130,8 @@ void testFindBySubjectApiKey() { var page = embeddingRepository.findBySubjectApiKey(model.getApiKey(), Pageable.unpaged()); assertThat(page.getTotalElements()).isEqualTo(5); - assertThat(page.getContent().stream().filter(p -> p.getSubjectName().equals(subject1.getSubjectName())).count()).isEqualTo(3); - assertThat(page.getContent().stream().filter(p -> p.getSubjectName().equals(subject2.getSubjectName())).count()).isEqualTo(2); + assertThat(page.getContent().stream().filter(p -> p.subjectName().equals(subject1.getSubjectName())).count()).isEqualTo(3); + assertThat(page.getContent().stream().filter(p -> p.subjectName().equals(subject2.getSubjectName())).count()).isEqualTo(2); } @Test diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/sdk/faces/service/FacesRestApiClientTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/sdk/faces/service/FacesRestApiClientTest.java index 1e7a31dc75..01a5cdca13 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/sdk/faces/service/FacesRestApiClientTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/sdk/faces/service/FacesRestApiClientTest.java @@ -63,10 +63,10 @@ void testFindFacesWithException(Class caughtClass, Class restApiClient.findFaces(photo, faceLimit, thresholdC, facePlugins); + Executable action = () -> restApiClient.findFaces(photo, faceLimit, thresholdC, facePlugins, true); // then assertThrows(thrownClass, action); @@ -80,10 +80,10 @@ void testFindFaces() { Integer faceLimit = 1; Double thresholdC = 1.0; String facePlugins = "plugins"; - when(feignClient.findFaces(photo, faceLimit, thresholdC, facePlugins)).thenReturn(expected); + when(feignClient.findFaces(photo, faceLimit, thresholdC, facePlugins, true)).thenReturn(expected); // when - FindFacesResponse actual = restApiClient.findFaces(photo, faceLimit, thresholdC, facePlugins); + FindFacesResponse actual = restApiClient.findFaces(photo, faceLimit, thresholdC, facePlugins, true); // then assertThat(actual, is(expected)); @@ -93,8 +93,8 @@ static Stream verifyFindFacesWithCalculator() { return Stream.of( Arguments.of(null, CALCULATOR_PLUGIN), Arguments.of("", CALCULATOR_PLUGIN), - Arguments.of("age,gender", CALCULATOR_PLUGIN + ",age,gender"), - Arguments.of(CALCULATOR_PLUGIN + ",age,gender", CALCULATOR_PLUGIN + ",age,gender") + Arguments.of("age,gender,pose", CALCULATOR_PLUGIN + ",age,gender,pose"), + Arguments.of(CALCULATOR_PLUGIN + ",age,gender,pose", CALCULATOR_PLUGIN + ",age,gender,pose") ); } @@ -106,10 +106,10 @@ void testFindFacesWithCalculator(String inPlugins, String outPlugins) { MultipartFile photo = mock(MultipartFile.class); Integer faceLimit = 1; Double thresholdC = 1.0; - when(feignClient.findFaces(photo, faceLimit, thresholdC, outPlugins)).thenReturn(expected); + when(feignClient.findFaces(photo, faceLimit, thresholdC, outPlugins, true)).thenReturn(expected); // when - FindFacesResponse actual = restApiClient.findFacesWithCalculator(photo, faceLimit, thresholdC, inPlugins); + FindFacesResponse actual = restApiClient.findFacesWithCalculator(photo, faceLimit, thresholdC, inPlugins, true); // then assertThat(actual, is(expected)); @@ -122,10 +122,10 @@ void testFindFacesWithCalculatorWithException(Class caughtC MultipartFile photo = mock(MultipartFile.class); Integer faceLimit = 1; Double thresholdC = 1.0; - when(feignClient.findFaces(photo, faceLimit, thresholdC, CALCULATOR_PLUGIN)).thenThrow(caughtClass); + when(feignClient.findFaces(photo, faceLimit, thresholdC, CALCULATOR_PLUGIN, true)).thenThrow(caughtClass); // when - Executable action = () -> restApiClient.findFacesWithCalculator(photo, faceLimit, thresholdC, null); + Executable action = () -> restApiClient.findFacesWithCalculator(photo, faceLimit, thresholdC, null, true); // then assertThrows(thrownClass, action); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingServiceTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingServiceTest.java index 18eb4c506b..7c8cabc76b 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingServiceTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingServiceTest.java @@ -1,6 +1,7 @@ package com.exadel.frs.core.trainservice.service; import com.exadel.frs.commonservice.entity.*; +import com.exadel.frs.commonservice.projection.EmbeddingProjection; import com.exadel.frs.core.trainservice.DbHelper; import com.exadel.frs.core.trainservice.EmbeddedPostgreSQLTest; import com.exadel.frs.core.trainservice.dao.SubjectDao; @@ -39,7 +40,24 @@ void testListEmbeddings() { // new EmbeddingInfo("calc", new double[]{1.0, 2.0}, img()) var size = 5; - final Page page = embeddingService.listEmbeddings(model.getApiKey(), PageRequest.of(0, size)); + final Page page = embeddingService.listEmbeddings(model.getApiKey(), null, PageRequest.of(0, size)); + + assertThat(page.getTotalElements(), is((long) count)); + assertThat(page.getTotalPages(), is(count / size)); + assertThat(page.getSize(), is(size)); + } + + @Test + void testListEmbeddingsWithSubjectName() { + final Model model = dbHelper.insertModel(); + + int count = 1; + var subjectName = "Johnny Depp"; + dbHelper.insertEmbeddingNoImg(dbHelper.insertSubject(model, subjectName)); + dbHelper.insertEmbeddingNoImg(dbHelper.insertSubject(model, "Not Johnny Depp")); + + var size = 1; + final Page page = embeddingService.listEmbeddings(model.getApiKey(), subjectName, PageRequest.of(0, size)); assertThat(page.getTotalElements(), is((long) count)); assertThat(page.getTotalPages(), is(count / size)); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingsRecognizeProcessServiceImplTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingsRecognizeProcessServiceImplTest.java new file mode 100644 index 0000000000..da00203214 --- /dev/null +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingsRecognizeProcessServiceImplTest.java @@ -0,0 +1,139 @@ +package com.exadel.frs.core.trainservice.service; + +import static com.exadel.frs.core.trainservice.system.global.Constants.PREDICTION_COUNT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.when; +import com.exadel.frs.commonservice.exception.IncorrectPredictionCountException; +import com.exadel.frs.commonservice.repository.EmbeddingRepository; +import com.exadel.frs.commonservice.sdk.faces.FacesApiClient; +import com.exadel.frs.core.trainservice.DbHelper; +import com.exadel.frs.core.trainservice.EmbeddedPostgreSQLTest; +import com.exadel.frs.core.trainservice.component.FaceClassifierPredictor; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; +import com.exadel.frs.core.trainservice.mapper.FacesMapper; +import com.exadel.frs.core.trainservice.repository.AppRepository; +import com.exadel.frs.core.trainservice.validation.ImageExtensionValidator; +import java.util.Collections; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.transaction.annotation.Transactional; + +@Transactional +class EmbeddingsRecognizeProcessServiceImplTest extends EmbeddedPostgreSQLTest { + + @Autowired + private DbHelper dbHelper; + + @Autowired + private ImageExtensionValidator imageExtensionValidator; + + @Autowired + private FacesMapper facesMapper; + + @Autowired + private AppRepository appRepository; + + @Autowired + private EmbeddingRepository embeddingRepository; + + @MockBean + private FacesApiClient facesApiClient; + + @MockBean + private FaceClassifierPredictor predictor; + + @Autowired + private EmbeddingsRecognizeProcessServiceImpl recognizeProcessService; + + @BeforeEach + void cleanUp() { + appRepository.deleteAll(); + appRepository.flush(); + } + + @Test + void processEmbeddings_TheInputEmbeddingExistsInTheDatabase_ShouldReturnCompleteSimilarity() { + var model = dbHelper.insertModel(); + var subject = dbHelper.insertSubject(model, "subject"); + var embedding = dbHelper.insertEmbeddingNoImg(subject); + + var params = ProcessEmbeddingsParams.builder() + .apiKey(model.getApiKey()) + .embeddings(new double[][]{embedding.getEmbedding()}) + .additionalParams(Collections.singletonMap(PREDICTION_COUNT, 1)) + .build(); + + when(predictor.predict(any(), any(), anyInt())).thenReturn(List.of(Pair.of(1.0, "subject"))); + assertThat(embeddingRepository.findAll()).containsOnly(embedding); + + var results = recognizeProcessService.processEmbeddings(params).getResult(); + + assertThat(embeddingRepository.findAll()).containsOnly(embedding); + assertThat(results).isNotEmpty().hasSize(1); + + var result = results.get(0); + + assertThat(result.getEmbedding()).isEqualTo(embedding.getEmbedding()); + assertThat(result.getSimilarities()).isNotEmpty().hasSize(1); + assertThat(result.getSimilarities().get(0).getSimilarity()).isEqualTo(1.0F); + assertThat(result.getSimilarities().get(0).getSubject()).isEqualTo("subject"); + } + + @Test + void processEmbeddings_TheInputEmbeddingDoesNotExistInTheDatabase_ShouldNotReturnCompleteSimilarity() { + var model = dbHelper.insertModel(); + var subject = dbHelper.insertSubject(model, "subject"); + var embedding = dbHelper.insertEmbeddingNoImg(subject); + + var params = ProcessEmbeddingsParams.builder() + .apiKey(model.getApiKey()) + .embeddings(new double[][]{new double[]{7.3, 8.4, 9.5}}) + .additionalParams(Collections.singletonMap(PREDICTION_COUNT, 1)) + .build(); + + when(predictor.predict(any(), any(), anyInt())).thenReturn(List.of(Pair.of(0.0, "subject"))); + assertThat(embeddingRepository.findAll()).containsOnly(embedding); + + var results = recognizeProcessService.processEmbeddings(params).getResult(); + + assertThat(embeddingRepository.findAll()).containsOnly(embedding); + assertThat(results).isNotEmpty().hasSize(1); + + var result = results.get(0); + + assertThat(result.getEmbedding()).isNotEqualTo(embedding.getEmbedding()); + assertThat(result.getSimilarities()).isNotEmpty().hasSize(1); + assertThat(result.getSimilarities().get(0).getSimilarity()).isEqualTo(0.0F); + assertThat(result.getSimilarities().get(0).getSubject()).isEqualTo("subject"); + } + + @ParameterizedTest + @ValueSource(ints = {0, -2}) + void processEmbeddings_PredictionCountIsIncorrect_ShouldThrowIncorrectPredictionCountException(int predictionCount) { + var params = ProcessEmbeddingsParams.builder() + .additionalParams(Collections.singletonMap(PREDICTION_COUNT, predictionCount)) + .build(); + + assertThatThrownBy(() -> recognizeProcessService.processEmbeddings(params)) + .isInstanceOf(IncorrectPredictionCountException.class); + } + + @Test + void processEmbeddings_PredictionCountIsNull_ShouldThrowIncorrectPredictionCountException() { + var params = ProcessEmbeddingsParams.builder() + .additionalParams(Collections.singletonMap(PREDICTION_COUNT, null)) + .build(); + + assertThatThrownBy(() -> recognizeProcessService.processEmbeddings(params)) + .isInstanceOf(IncorrectPredictionCountException.class); + } +} diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingsVerificationProcessServiceImplTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingsVerificationProcessServiceImplTest.java new file mode 100644 index 0000000000..4f131429a1 --- /dev/null +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/EmbeddingsVerificationProcessServiceImplTest.java @@ -0,0 +1,100 @@ +package com.exadel.frs.core.trainservice.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.when; +import com.exadel.frs.commonservice.exception.WrongEmbeddingCountException; +import com.exadel.frs.commonservice.sdk.faces.FacesApiClient; +import com.exadel.frs.core.trainservice.DbHelper; +import com.exadel.frs.core.trainservice.EmbeddedPostgreSQLTest; +import com.exadel.frs.core.trainservice.component.FaceClassifierPredictor; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; +import com.exadel.frs.core.trainservice.mapper.FacesMapper; +import com.exadel.frs.core.trainservice.repository.AppRepository; +import com.exadel.frs.core.trainservice.validation.ImageExtensionValidator; +import org.apache.commons.lang3.ArrayUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.transaction.annotation.Transactional; + +@Transactional +class EmbeddingsVerificationProcessServiceImplTest extends EmbeddedPostgreSQLTest { + + @Autowired + private DbHelper dbHelper; + + @Autowired + private ImageExtensionValidator imageExtensionValidator; + + @Autowired + private FacesMapper facesMapper; + + @Autowired + private AppRepository appRepository; + + @MockBean + private FaceClassifierPredictor predictor; + + @MockBean + private FacesApiClient facesApiClient; + + @Autowired + private EmbeddingsVerificationProcessServiceImpl verificationProcessService; + + @BeforeEach + void cleanUp() { + appRepository.deleteAll(); + appRepository.flush(); + } + + @Test + void processEmbeddings_ThereAreTwoEmbeddingsInTheDatabase_ShouldReturnTwoSimilarityResultInSortedOrder() { + var source = new double[]{1.0, 2.0, 3.0}; + var targets = new double[][]{ + new double[]{4.0, 5.0, 6.0}, + new double[]{7.0, 8.0, 9.0} + }; + var similarities = new double[]{0.3, 0.5}; + var params = buildParams(source, targets); + + when(predictor.verify(source, targets)).thenReturn(similarities); + + var results = verificationProcessService.processEmbeddings(params).getResult(); + + assertThat(results).isNotEmpty().hasSize(2); + + var result1 = results.get(0); + var result2 = results.get(1); + + assertThat(result1.getSimilarity()).isEqualTo(0.5F); + assertThat(result2.getSimilarity()).isEqualTo(0.3F); + assertThat(result1.getEmbedding()).isEqualTo(targets[1]); + assertThat(result2.getEmbedding()).isEqualTo(targets[0]); + } + + @Test + void processEmbeddings_TooFewTargets_ShouldThrowWrongEmbeddingCountException() { + var source = new double[]{1.0, 2.0, 3.0}; + var targets = new double[0][]; + var params = buildParams(source, targets); + + assertThatThrownBy(() -> verificationProcessService.processEmbeddings(params)) + .isInstanceOf(WrongEmbeddingCountException.class); + } + + @Test + void processEmbeddings_EmbeddingsAreNull_ShouldThrowWrongEmbeddingCountException() { + var params = ProcessEmbeddingsParams.builder().build(); + + assertThatThrownBy(() -> verificationProcessService.processEmbeddings(params)) + .isInstanceOf(WrongEmbeddingCountException.class); + } + + private ProcessEmbeddingsParams buildParams(double[] source, double[][] targets) { + return ProcessEmbeddingsParams.builder() + .embeddings(ArrayUtils.insert(0, targets, source)) + .build(); + } +} diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/ModelServiceTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/ModelServiceTest.java index 6615d9a71f..ea1060b351 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/ModelServiceTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/ModelServiceTest.java @@ -51,12 +51,14 @@ void setUp() { @Test void validateModelKeyOkValidationResult() { - when(modelRepository.findByApiKeyAndType(MODEL_KEY, MODEL_TYPE)).thenReturn(Optional.of(new Model())); + val model = Model.builder().id(1L).build(); + + when(modelRepository.findByApiKeyAndType(MODEL_KEY, MODEL_TYPE)).thenReturn(Optional.of(model)); val actual = modelService.validateModelKey(MODEL_KEY, MODEL_TYPE); assertThat(actual).isNotNull(); - assertThat(actual).isEqualTo(OK); + assertThat(actual.getResult()).isEqualTo(OK); verify(modelRepository).findByApiKeyAndType(MODEL_KEY, MODEL_TYPE); verifyNoMoreInteractions(modelRepository); @@ -69,7 +71,7 @@ void validateModelKeyForbiddenValidationResult() { val actual = modelService.validateModelKey(MODEL_KEY, MODEL_TYPE); assertThat(actual).isNotNull(); - assertThat(actual).isEqualTo(FORBIDDEN); + assertThat(actual.getResult()).isEqualTo(FORBIDDEN); verify(modelRepository).findByApiKeyAndType(MODEL_KEY, MODEL_TYPE); verifyNoMoreInteractions(modelRepository); diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/ModelStatisticServiceTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/ModelStatisticServiceTest.java new file mode 100644 index 0000000000..49c92e449f --- /dev/null +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/ModelStatisticServiceTest.java @@ -0,0 +1,327 @@ +package com.exadel.frs.core.trainservice.service; + +import static java.time.LocalDateTime.now; +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.HOURS; +import static org.assertj.core.api.Assertions.assertThat; +import com.exadel.frs.commonservice.repository.ModelStatisticRepository; +import com.exadel.frs.core.trainservice.DbHelper; +import com.exadel.frs.core.trainservice.EmbeddedPostgreSQLTest; +import com.exadel.frs.core.trainservice.cache.ModelStatisticCacheProvider; +import com.exadel.frs.core.trainservice.util.CronExecution; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import javax.transaction.Transactional; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +@Transactional +class ModelStatisticServiceTest extends EmbeddedPostgreSQLTest { + + @Autowired + private DbHelper dbHelper; + + @Autowired + private ModelStatisticCacheProvider statisticCacheProvider; + + @Autowired + private ModelStatisticRepository statisticRepository; + + @Autowired + private ModelStatisticService statisticService; + + private final CronExecution cronExecution = new CronExecution("0 0 * ? * *"); + + @Test + void shouldTerminateExecutionWithoutUpdateOrRecordStatistic() { + assertThat(statisticRepository.count()).isZero(); + + statisticService.updateAndRecordStatistics(); + + assertThat(statisticRepository.count()).isZero(); + } + + @Test + void shouldRecordOneStatistic() { + var model = dbHelper.insertModel(); + + statisticCacheProvider.incrementRequestCount(model.getId()); + statisticCacheProvider.incrementRequestCount(model.getId()); + statisticCacheProvider.incrementRequestCount(model.getId()); + + assertThat(statisticRepository.count()).isZero(); + + statisticService.updateAndRecordStatistics(); + + assertThat(statisticRepository.count()).isEqualTo(1); + + var statistics = statisticRepository.findAll(); + var actual = statistics.get(0); + + assertThat(actual).isNotNull(); + assertThat(actual.getId()).isPositive(); + assertThat(actual.getCreatedDate()).isBefore(now(UTC)); + assertThat(actual.getRequestCount()).isEqualTo(3); + assertThat(actual.getModel().getId()).isEqualTo(model.getId()); + } + + @Test + void shouldRecordThreeStatistics() { + var model1 = dbHelper.insertModel(); + var model2 = dbHelper.insertModel(); + var model3 = dbHelper.insertModel(); + + statisticCacheProvider.incrementRequestCount(model1.getId()); + statisticCacheProvider.incrementRequestCount(model2.getId()); + statisticCacheProvider.incrementRequestCount(model2.getId()); + statisticCacheProvider.incrementRequestCount(model3.getId()); + statisticCacheProvider.incrementRequestCount(model3.getId()); + statisticCacheProvider.incrementRequestCount(model3.getId()); + + assertThat(statisticRepository.count()).isZero(); + + statisticService.updateAndRecordStatistics(); + + assertThat(statisticRepository.count()).isEqualTo(3); + + var statistics = statisticRepository.findAll(); + + var optional1 = statistics.stream().filter(e -> e.getModel().getId().equals(model1.getId())).findFirst(); + var optional2 = statistics.stream().filter(e -> e.getModel().getId().equals(model2.getId())).findFirst(); + var optional3 = statistics.stream().filter(e -> e.getModel().getId().equals(model3.getId())).findFirst(); + + assertThat(optional1).isPresent(); + assertThat(optional2).isPresent(); + assertThat(optional3).isPresent(); + + var actual1 = optional1.get(); + var actual2 = optional2.get(); + var actual3 = optional3.get(); + + assertThat(actual1.getId()).isPositive(); + assertThat(actual1.getCreatedDate()).isBefore(now(UTC)); + assertThat(actual1.getRequestCount()).isEqualTo(1); + assertThat(actual1.getModel().getId()).isEqualTo(model1.getId()); + + assertThat(actual2.getId()).isPositive(); + assertThat(actual2.getCreatedDate()).isBefore(now(UTC)); + assertThat(actual2.getRequestCount()).isEqualTo(2); + assertThat(actual2.getModel().getId()).isEqualTo(model2.getId()); + + assertThat(actual3.getId()).isPositive(); + assertThat(actual3.getCreatedDate()).isBefore(now(UTC)); + assertThat(actual3.getRequestCount()).isEqualTo(3); + assertThat(actual3.getModel().getId()).isEqualTo(model3.getId()); + } + + @Test + void shouldUpdateOneStatistic() { + var model = dbHelper.insertModel(); + var statistic = dbHelper.insertModelStatistic(model, 5, getCreateDate()); + + statisticCacheProvider.incrementRequestCount(model.getId()); + statisticCacheProvider.incrementRequestCount(model.getId()); + statisticCacheProvider.incrementRequestCount(model.getId()); + + assertThat(statisticRepository.count()).isEqualTo(1); + + statisticService.updateAndRecordStatistics(); + + assertThat(statisticRepository.count()).isEqualTo(1); + + var statistics = statisticRepository.findAll(); + var actual = statistics.get(0); + + assertThat(actual.getId()).isPositive(); + assertThat(actual.getCreatedDate()).isBefore(now(UTC)); + assertThat(actual.getRequestCount()).isEqualTo(8); + assertThat(actual.getModel().getId()).isEqualTo(model.getId()); + } + + @Test + void shouldUpdateThreeStatistics() { + var model1 = dbHelper.insertModel(); + var model2 = dbHelper.insertModel(); + var model3 = dbHelper.insertModel(); + + var statistic1 = dbHelper.insertModelStatistic(model1, 1, getCreateDate()); + var statistic2 = dbHelper.insertModelStatistic(model2, 2, getCreateDate()); + var statistic3 = dbHelper.insertModelStatistic(model3, 3, getCreateDate()); + + statisticCacheProvider.incrementRequestCount(model1.getId()); + statisticCacheProvider.incrementRequestCount(model2.getId()); + statisticCacheProvider.incrementRequestCount(model2.getId()); + statisticCacheProvider.incrementRequestCount(model3.getId()); + statisticCacheProvider.incrementRequestCount(model3.getId()); + statisticCacheProvider.incrementRequestCount(model3.getId()); + + assertThat(statisticRepository.count()).isEqualTo(3); + + statisticService.updateAndRecordStatistics(); + + assertThat(statisticRepository.count()).isEqualTo(3); + + var statistics = statisticRepository.findAll(); + + var optional1 = statistics.stream().filter(e -> e.getModel().getId().equals(model1.getId())).findFirst(); + var optional2 = statistics.stream().filter(e -> e.getModel().getId().equals(model2.getId())).findFirst(); + var optional3 = statistics.stream().filter(e -> e.getModel().getId().equals(model3.getId())).findFirst(); + + assertThat(optional1).isPresent(); + assertThat(optional2).isPresent(); + assertThat(optional3).isPresent(); + + var actual1 = optional1.get(); + var actual2 = optional2.get(); + var actual3 = optional3.get(); + + assertThat(actual1.getId()).isPositive(); + assertThat(actual1.getCreatedDate()).isBefore(now(UTC)); + assertThat(actual1.getRequestCount()).isEqualTo(2); + assertThat(actual1.getModel().getId()).isEqualTo(model1.getId()); + + assertThat(actual2.getId()).isPositive(); + assertThat(actual2.getCreatedDate()).isBefore(now(UTC)); + assertThat(actual2.getRequestCount()).isEqualTo(4); + assertThat(actual2.getModel().getId()).isEqualTo(model2.getId()); + + assertThat(actual3.getId()).isPositive(); + assertThat(actual3.getCreatedDate()).isBefore(now(UTC)); + assertThat(actual3.getRequestCount()).isEqualTo(6); + assertThat(actual3.getModel().getId()).isEqualTo(model3.getId()); + } + + @Test + void shouldUpdateOneStatisticAndRecordOneStatistic() { + var model1 = dbHelper.insertModel(); + var model2 = dbHelper.insertModel(); + + var statisticToUpdate = dbHelper.insertModelStatistic(model1, 1, getCreateDate()); + + statisticCacheProvider.incrementRequestCount(model1.getId()); + statisticCacheProvider.incrementRequestCount(model1.getId()); + statisticCacheProvider.incrementRequestCount(model2.getId()); + statisticCacheProvider.incrementRequestCount(model2.getId()); + + assertThat(statisticRepository.count()).isEqualTo(1); + + statisticService.updateAndRecordStatistics(); + + assertThat(statisticRepository.count()).isEqualTo(2); + + var statistics = statisticRepository.findAll(); + + var updatedStatisticOptional = statistics.stream().filter(e -> e.getModel().getId().equals(model1.getId())).findFirst(); + var recordedStatisticOptional = statistics.stream().filter(e -> e.getModel().getId().equals(model2.getId())).findFirst(); + + assertThat(updatedStatisticOptional).isPresent(); + assertThat(recordedStatisticOptional).isPresent(); + + var updatedStatistic = updatedStatisticOptional.get(); + var recordedStatistic = recordedStatisticOptional.get(); + + assertThat(updatedStatistic.getId()).isPositive(); + assertThat(updatedStatistic.getCreatedDate()).isBefore(now(UTC)); + assertThat(updatedStatistic.getRequestCount()).isEqualTo(3); + assertThat(updatedStatistic.getModel().getId()).isEqualTo(model1.getId()); + + assertThat(recordedStatistic.getId()).isPositive(); + assertThat(recordedStatistic.getCreatedDate()).isBefore(now(UTC)); + assertThat(recordedStatistic.getRequestCount()).isEqualTo(2); + assertThat(recordedStatistic.getModel().getId()).isEqualTo(model2.getId()); + } + + @Test + void shouldUpdateThreeStatisticsAndRecordThreeStatistics() { + var model1 = dbHelper.insertModel(); + var model2 = dbHelper.insertModel(); + var model3 = dbHelper.insertModel(); + var model4 = dbHelper.insertModel(); + var model5 = dbHelper.insertModel(); + var model6 = dbHelper.insertModel(); + + var statisticToUpdate1 = dbHelper.insertModelStatistic(model1, 1, getCreateDate()); + var statisticToUpdate2 = dbHelper.insertModelStatistic(model2, 2, getCreateDate()); + var statisticToUpdate3 = dbHelper.insertModelStatistic(model3, 3, getCreateDate()); + + statisticCacheProvider.incrementRequestCount(model1.getId()); + statisticCacheProvider.incrementRequestCount(model1.getId()); + statisticCacheProvider.incrementRequestCount(model2.getId()); + statisticCacheProvider.incrementRequestCount(model2.getId()); + statisticCacheProvider.incrementRequestCount(model3.getId()); + statisticCacheProvider.incrementRequestCount(model3.getId()); + + statisticCacheProvider.incrementRequestCount(model4.getId()); + statisticCacheProvider.incrementRequestCount(model5.getId()); + statisticCacheProvider.incrementRequestCount(model5.getId()); + statisticCacheProvider.incrementRequestCount(model6.getId()); + statisticCacheProvider.incrementRequestCount(model6.getId()); + statisticCacheProvider.incrementRequestCount(model6.getId()); + + assertThat(statisticRepository.count()).isEqualTo(3); + + statisticService.updateAndRecordStatistics(); + + assertThat(statisticRepository.count()).isEqualTo(6); + + var statistics = statisticRepository.findAll(); + + var updatedStatisticOptional1 = statistics.stream().filter(e -> e.getModel().getId().equals(model1.getId())).findFirst(); + var updatedStatisticOptional2 = statistics.stream().filter(e -> e.getModel().getId().equals(model2.getId())).findFirst(); + var updatedStatisticOptional3 = statistics.stream().filter(e -> e.getModel().getId().equals(model3.getId())).findFirst(); + var recordedStatisticOptional1 = statistics.stream().filter(e -> e.getModel().getId().equals(model4.getId())).findFirst(); + var recordedStatisticOptional2 = statistics.stream().filter(e -> e.getModel().getId().equals(model5.getId())).findFirst(); + var recordedStatisticOptional3 = statistics.stream().filter(e -> e.getModel().getId().equals(model6.getId())).findFirst(); + + assertThat(updatedStatisticOptional1).isPresent(); + assertThat(updatedStatisticOptional2).isPresent(); + assertThat(updatedStatisticOptional3).isPresent(); + assertThat(recordedStatisticOptional1).isPresent(); + assertThat(recordedStatisticOptional2).isPresent(); + assertThat(recordedStatisticOptional3).isPresent(); + + var updatedStatistic1 = updatedStatisticOptional1.get(); + var updatedStatistic2 = updatedStatisticOptional2.get(); + var updatedStatistic3 = updatedStatisticOptional3.get(); + var recordedStatistic1 = recordedStatisticOptional1.get(); + var recordedStatistic2 = recordedStatisticOptional2.get(); + var recordedStatistic3 = recordedStatisticOptional3.get(); + + assertThat(updatedStatistic1.getId()).isPositive(); + assertThat(updatedStatistic1.getCreatedDate()).isBefore(now(UTC)); + assertThat(updatedStatistic1.getRequestCount()).isEqualTo(3); + assertThat(updatedStatistic1.getModel().getId()).isEqualTo(model1.getId()); + + assertThat(updatedStatistic2.getId()).isPositive(); + assertThat(updatedStatistic2.getCreatedDate()).isBefore(now(UTC)); + assertThat(updatedStatistic2.getRequestCount()).isEqualTo(4); + assertThat(updatedStatistic2.getModel().getId()).isEqualTo(model2.getId()); + + assertThat(updatedStatistic3.getId()).isPositive(); + assertThat(updatedStatistic3.getCreatedDate()).isBefore(now(UTC)); + assertThat(updatedStatistic3.getRequestCount()).isEqualTo(5); + assertThat(updatedStatistic3.getModel().getId()).isEqualTo(model3.getId()); + + assertThat(recordedStatistic1.getId()).isPositive(); + assertThat(recordedStatistic1.getCreatedDate()).isBefore(now(UTC)); + assertThat(recordedStatistic1.getRequestCount()).isEqualTo(1); + assertThat(recordedStatistic1.getModel().getId()).isEqualTo(model4.getId()); + + assertThat(recordedStatistic2.getId()).isPositive(); + assertThat(recordedStatistic2.getCreatedDate()).isBefore(now(UTC)); + assertThat(recordedStatistic2.getRequestCount()).isEqualTo(2); + assertThat(recordedStatistic2.getModel().getId()).isEqualTo(model5.getId()); + + assertThat(recordedStatistic3.getId()).isPositive(); + assertThat(recordedStatistic3.getCreatedDate()).isBefore(now(UTC)); + assertThat(recordedStatistic3.getRequestCount()).isEqualTo(3); + assertThat(recordedStatistic3.getModel().getId()).isEqualTo(model6.getId()); + } + + private LocalDateTime getCreateDate() { + return cronExecution.getLastExecutionBefore(ZonedDateTime.now(UTC)) + .flatMap(cronExecution::getLastExecutionBefore) + .map(last -> last.toLocalDateTime().truncatedTo(HOURS)) + .orElse(null); + } +} diff --git a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/SubjectServiceTest.java b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/SubjectServiceTest.java index 79fb8ea0d4..2d43962615 100644 --- a/java/api/src/test/java/com/exadel/frs/core/trainservice/service/SubjectServiceTest.java +++ b/java/api/src/test/java/com/exadel/frs/core/trainservice/service/SubjectServiceTest.java @@ -16,10 +16,26 @@ package com.exadel.frs.core.trainservice.service; +import static com.exadel.frs.core.trainservice.ItemsBuilder.makeEnhancedEmbeddingProjection; +import static com.exadel.frs.core.trainservice.service.SubjectService.MAX_FACES_TO_RECOGNIZE; +import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_ID; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; import com.exadel.frs.commonservice.dto.ExecutionTimeDto; import com.exadel.frs.commonservice.entity.Embedding; import com.exadel.frs.commonservice.entity.Subject; +import com.exadel.frs.commonservice.exception.IncorrectImageIdException; import com.exadel.frs.commonservice.exception.TooManyFacesException; +import com.exadel.frs.commonservice.exception.WrongEmbeddingCountException; import com.exadel.frs.commonservice.sdk.faces.FacesApiClient; import com.exadel.frs.commonservice.sdk.faces.feign.dto.FacesBox; import com.exadel.frs.commonservice.sdk.faces.feign.dto.FindFacesResponse; @@ -30,8 +46,16 @@ import com.exadel.frs.core.trainservice.component.FaceClassifierPredictor; import com.exadel.frs.core.trainservice.component.classifiers.EuclideanDistanceClassifier; import com.exadel.frs.core.trainservice.dao.SubjectDao; +import com.exadel.frs.core.trainservice.dto.EmbeddingVerificationProcessResult; +import com.exadel.frs.core.trainservice.dto.ProcessEmbeddingsParams; import com.exadel.frs.core.trainservice.dto.ProcessImageParams; -import com.exadel.frs.core.trainservice.mapper.FacesMapper; +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -39,29 +63,11 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; -import org.mapstruct.factory.Mappers; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Spy; import org.springframework.mock.web.MockMultipartFile; import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static com.exadel.frs.core.trainservice.ItemsBuilder.makeEmbedding; -import static com.exadel.frs.core.trainservice.service.SubjectService.MAX_FACES_TO_RECOGNIZE; -import static com.exadel.frs.core.trainservice.system.global.Constants.IMAGE_ID; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; - class SubjectServiceTest { private static final String API_KEY = "apiKey"; @@ -72,9 +78,6 @@ class SubjectServiceTest { @Mock private FacesApiClient facesApiClient; - @Spy - private FacesMapper facesMapper = Mappers.getMapper(FacesMapper.class); - @Mock private EmbeddingCacheProvider embeddingCacheProvider; @@ -182,7 +185,7 @@ void testSaveCalculatedEmbedding() throws IOException { var detProbThreshold = 0.7; MultipartFile file = new MockMultipartFile("anyname", new byte[]{0xA}); - when(facesApiClient.findFacesWithCalculator(file, MAX_FACES_TO_RECOGNIZE, detProbThreshold, null)) + when(facesApiClient.findFacesWithCalculator(file, MAX_FACES_TO_RECOGNIZE, detProbThreshold, null, true)) .thenReturn(findFacesResponse(1)); when(euclideanDistanceClassifier.normalizeOne(any())) .thenReturn(new double[]{1.1, 2.2}); @@ -200,7 +203,7 @@ void tooManyFacesFound() { var detProbThreshold = 0.7; MultipartFile file = new MockMultipartFile("anyname", new byte[]{0xA}); - when(facesApiClient.findFacesWithCalculator(file, MAX_FACES_TO_RECOGNIZE, detProbThreshold, null)) + when(facesApiClient.findFacesWithCalculator(file, MAX_FACES_TO_RECOGNIZE, detProbThreshold, null, true)) .thenReturn(findFacesResponse(3)); assertThatThrownBy(() -> @@ -214,17 +217,18 @@ void tooManyFacesFound() { @ValueSource(booleans = {true, false}) void testVerifyFaces(boolean status) { var detProbThreshold = 0.7; - MultipartFile file = new MockMultipartFile("anyname", new byte[]{0xA}); + var randomUUId = UUID.randomUUID(); + var file = new MockMultipartFile("anyname", new byte[]{0xA}); + var embeddingCollection = mock(EmbeddingCollection.class); - when(facesApiClient.findFacesWithCalculator(any(), any(), any(), any())) + when(facesApiClient.findFacesWithCalculator(any(), any(), any(), any(), any())) .thenReturn(findFacesResponse(2)); when(embeddingCacheProvider.getOrLoad(API_KEY)) - .thenReturn(EmbeddingCollection.from(Stream.of( - makeEmbedding("A", API_KEY), - makeEmbedding("B", API_KEY) - ))); + .thenReturn(embeddingCollection); when(classifierPredictor.verify(any(), any(), any())) .thenReturn(0.0); + when(embeddingCollection.getSubjectNameByEmbeddingId(randomUUId)) + .thenReturn(Optional.of("A")); var result = subjectService.verifyFace( ProcessImageParams.builder() @@ -233,7 +237,7 @@ void testVerifyFaces(boolean status) { .limit(MAX_FACES_TO_RECOGNIZE) .detProbThreshold(detProbThreshold) .status(status) - .additionalParams(Map.of(IMAGE_ID, UUID.randomUUID())) + .additionalParams(Map.of(IMAGE_ID, randomUUId)) .build() ); @@ -249,6 +253,75 @@ void testVerifyFaces(boolean status) { } } + @Test + void verifyEmbedding_ThereAreTwoTargetsAndOneSourceInTheDatabase_ShouldReturnTwoSimilarityResultsInSortedOrder() { + var targets = new double[][]{ + new double[]{1, 2, 3}, + new double[]{4, 5, 6} + }; + var sourceId = UUID.randomUUID(); + var apiKey = UUID.randomUUID().toString(); + + var params = ProcessEmbeddingsParams.builder() + .apiKey(apiKey) + .embeddings(targets) + .additionalParams(Map.of(IMAGE_ID, sourceId)) + .build(); + + when(classifierPredictor.verify(apiKey, targets[0], sourceId)).thenReturn(0.5); + when(classifierPredictor.verify(apiKey, targets[1], sourceId)).thenReturn(1.0); + + var results = subjectService.verifyEmbedding(params).getResult(); + + assertThat(results).isNotEmpty().hasSize(2); + + var result1 = results.get(0); + var result2 = results.get(1); + + assertThat(result1.getSimilarity()).isEqualTo(1.0F); + assertThat(result2.getSimilarity()).isEqualTo(0.5F); + assertThat(result1.getEmbedding()).isEqualTo(targets[1]); + assertThat(result2.getEmbedding()).isEqualTo(targets[0]); + } + + @Test + void verifyEmbedding_ThereAreNoTargets_ShouldThrowWrongEmbeddingCountException() { + var params = ProcessEmbeddingsParams.builder() + .embeddings(new double[][]{}) + .build(); + + assertThatThrownBy(() -> subjectService.verifyEmbedding(params)) + .isInstanceOf(WrongEmbeddingCountException.class); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testInvalidImageIdException(boolean status){ + var detProbThreshold = 0.7; + var randomUUId = UUID.randomUUID(); + var file = new MockMultipartFile("anyname", new byte[]{0xA}); + var embeddingCollection = EmbeddingCollection.from(Stream.of( + makeEnhancedEmbeddingProjection("A"), + makeEnhancedEmbeddingProjection("B"))); + + when(facesApiClient.findFacesWithCalculator(any(), any(), any(), any(), any())) + .thenReturn(findFacesResponse(2)); + when(embeddingCacheProvider.getOrLoad(API_KEY)) + .thenReturn(embeddingCollection); + when(classifierPredictor.verify(any(), any(), any())) + .thenReturn(0.0); + assertThrows(IncorrectImageIdException.class, ()-> subjectService.verifyFace( + ProcessImageParams.builder() + .apiKey(API_KEY) + .file(file) + .limit(MAX_FACES_TO_RECOGNIZE) + .detProbThreshold(detProbThreshold) + .status(status) + .additionalParams(Map.of(IMAGE_ID, randomUUId)) + .build() + )); + } + private static FindFacesResponse findFacesResponse(int faceCount) { return FindFacesResponse.builder() .result( diff --git a/java/api/src/test/resources/application.yml b/java/api/src/test/resources/application.yml index 449bf963a6..99df715f8d 100644 --- a/java/api/src/test/resources/application.yml +++ b/java/api/src/test/resources/application.yml @@ -5,16 +5,22 @@ spring: enabled: false liquibase: enabled: false - datasource-pg: - driver-class-name: org.postgresql.Driver - url: ${POSTGRES_URL:jdbc:postgresql://compreface-postgres-db:5432/frs} - username: ${POSTGRES_USER:postgres} - password: ${POSTGRES_PASSWORD:postgres} + parameters: + common-client: + client-id: CommonClientId + access-token-validity: 2400 + refresh-token-validity: 1209600 + authorized-grant-types: password,refresh_token jpa: properties: hibernate: default_schema: public - jdbc.lob.non_contextual_creation: true # fix for Caused by: java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgConnection.createClob() is not yet implemented. + jdbc: + lob.non_contextual_creation: true # fix for Caused by: java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgConnection.createClob() is not yet implemented. + batch_size: 10 + order_inserts: true + order_updates: true + batch_versioned_data: true format_sql: true dialect: org.hibernate.dialect.PostgreSQL10Dialect show_sql: true @@ -42,8 +48,18 @@ app: appery-io: url: https://localhost/rest/1/db/collections api-key: ${APPERY_API_KEY:#{null}} + faces: + connect-timeout: ${CONNECTION_TIMEOUT:10000} + read-timeout: ${READ_TIMEOUT:60000} + retryer: + max-attempts: ${MAX_ATTEMPTS:1} + +statistic: + model: + cron-expression: ${MODEL_STATISTIC_CRON_EXPRESSION:0 0 * ? * *} + logging: level: org.hibernate.SQL: DEBUG org.hibernate.type.descriptor.sql.BasicBinder: TRACE - org.hibernate.type: TRACE \ No newline at end of file + org.hibernate.type: TRACE diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.1.8.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.1.8.yaml new file mode 100644 index 0000000000..eb5f0614a8 --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.1.8.yaml @@ -0,0 +1,52 @@ +databaseChangeLog: + - changeSet: + id: add-detection-verification-services + author: Shreyansh Sancheti + preConditions: + - onFail: MARK_RAN + - sqlCheck: + expectedResult: 1 + sql: select count(1) from app where id=0 + changes: + - insert: + tableName: model + columns: + - column: + name: id + value: 9223372036854775807 + - column: + name: name + value: Detection_Demo + - column: + name: type + value: D + - column: + name: guid + value: 00000000-0000-0000-0000-000000000003 + - column: + name: app_id + value: 0 + - column: + name: api_key + value: 00000000-0000-0000-0000-000000000003 + - insert: + tableName: model + columns: + - column: + name: id + value: 9223372036854775806 + - column: + name: name + value: Verification_Demo + - column: + name: type + value: V + - column: + name: guid + value: 00000000-0000-0000-0000-000000000004 + - column: + name: app_id + value: 0 + - column: + name: api_key + value: 00000000-0000-0000-0000-000000000004 \ No newline at end of file diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.1.9.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.1.9.yaml new file mode 100644 index 0000000000..60669c5a30 --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.1.9.yaml @@ -0,0 +1,18 @@ +databaseChangeLog: + - changeSet: + id: add-created_date-field-to-model-entity + author: Khasan Sidikov + changes: + - addColumn: + tableName: model + columns: + - column: + name: created_date + type: timestamp + - changeSet: + id: update-model-table-created_date-column + author: Khasan Sidikov + changes: + - sql: + comment: Update created_date to now() + sql: UPDATE model SET created_date=now() WHERE created_date IS NULL; \ No newline at end of file diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.2.0.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.2.0.yaml new file mode 100644 index 0000000000..df2a79461a --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.2.0.yaml @@ -0,0 +1,50 @@ +databaseChangeLog: + - changeSet: + id: create-model-statistic-table + author: Volodymyr Bushko + changes: + # model_statistic + - createTable: + tableName: model_statistic + columns: + - column: + name: id + type: bigint + - column: + name: request_count + type: int + - column: + name: model_id + type: bigint + - column: + name: created_date + type: timestamp + + - addPrimaryKey: + columnNames: id + constraintName: pk_model_statistic + tableName: model_statistic + + - addForeignKeyConstraint: + baseColumnNames: model_id + baseTableName: model_statistic + referencedColumnNames: id + referencedTableName: model + constraintName: fk_model_id + onDelete: CASCADE + onUpdate: CASCADE + + - addNotNullConstraint: + tableName: model_statistic + columnName: request_count + + - addNotNullConstraint: + tableName: model_statistic + columnName: model_id + + - addNotNullConstraint: + tableName: model_statistic + columnName: created_date + + - createSequence: + sequenceName: model_statistic_id_seq diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.2.1.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.2.1.yaml new file mode 100644 index 0000000000..d310e97571 --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.2.1.yaml @@ -0,0 +1,43 @@ +databaseChangeLog: + - changeSet: + id: create-table-lock-table + author: Volodymyr Bushko + changes: + # table_lock + - createTable: + tableName: table_lock + columns: + - column: + name: id + type: uuid + - column: + name: lock_name + type: varchar(50) + + - addPrimaryKey: + columnNames: id + constraintName: pk_table_lock + tableName: table_lock + + - addNotNullConstraint: + tableName: table_lock + columnName: lock_name + + - addUniqueConstraint: + columnNames: lock_name + constraintName: table_lock_lock_name_uindex + tableName: table_lock + + - changeSet: + id: insert-into-table-lock-model_statistic_lock-row + author: Volodymyr Bushko + changes: + - insert: + tableName: table_lock + columns: + - column: + name: id + value: 00000000-0000-0000-0000-000000000001 + - column: + name: lock_name + value: MODEL_STATISTIC_LOCK diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.2.2.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.2.2.yaml new file mode 100644 index 0000000000..28cc025914 --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.2.2.yaml @@ -0,0 +1,13 @@ +databaseChangeLog: + - changeSet: + id: drop-app_model-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: app_model + changes: + - dropTable: + schemaName: public + tableName: app_model diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.2.3.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.2.3.yaml new file mode 100644 index 0000000000..517806c34f --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.2.3.yaml @@ -0,0 +1,45 @@ +databaseChangeLog: + - changeSet: + id: create-reset_password_token-table + author: Volodymyr Bushko + changes: + # reset_password_token + - createTable: + tableName: reset_password_token + columns: + - column: + name: token + type: uuid + - column: + name: user_email + type: varchar(63) + - column: + name: expires_in + type: timestamp + + - addPrimaryKey: + columnNames: token + tableName: reset_password_token + constraintName: pk_reset_password_token + + - addForeignKeyConstraint: + baseColumnNames: user_email + baseTableName: reset_password_token + referencedColumnNames: email + referencedTableName: user + constraintName: fk_reset_password_token_user_email + onDelete: CASCADE + onUpdate: CASCADE + + - addNotNullConstraint: + columnName: user_email + tableName: reset_password_token + + - addNotNullConstraint: + columnName: expires_in + tableName: reset_password_token + + - addUniqueConstraint: + columnNames: user_email + tableName: reset_password_token + constraintName: reset_password_token_user_email_uindex diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.2.4.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.2.4.yaml new file mode 100644 index 0000000000..959452820e --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.2.4.yaml @@ -0,0 +1,23 @@ +databaseChangeLog: + - changeSet: + id: remove-special-characters-from-names + author: Volodymyr Bushko + changes: + - customChange: { + "class": "com.exadel.frs.commonservice.system.liquibase.customchange.RemoveSpecialCharactersCustomChange", + "table": "app", + "primaryKeyColumn": "id", + "targetColumn": "name" + } + - customChange: { + "class": "com.exadel.frs.commonservice.system.liquibase.customchange.RemoveSpecialCharactersCustomChange", + "table": "model", + "primaryKeyColumn": "id", + "targetColumn": "name" + } + - customChange: { + "class": "com.exadel.frs.commonservice.system.liquibase.customchange.RemoveSpecialCharactersCustomChange", + "table": "subject", + "primaryKeyColumn": "id", + "targetColumn": "subject_name" + } diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.2.5.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.2.5.yaml new file mode 100644 index 0000000000..04b8b0ddb9 --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.2.5.yaml @@ -0,0 +1,33 @@ +databaseChangeLog: + - changeSet: + id: expand-oauth_access_token-and-oauth_refresh_token-with-expiration-column + author: Volodymyr Bushko + changes: + # add an expiration column to the oauth_access_token & oauth_refresh_token tables + - addColumn: + tableName: oauth_access_token + columns: + - column: + name: expiration + type: timestamp + - addColumn: + tableName: oauth_refresh_token + columns: + - column: + name: expiration + type: timestamp + # set the expiration columns in order to avoid conflicts + - customChange: { + "class": "com.exadel.frs.commonservice.system.liquibase.customchange.SetOAuthTokenExpirationCustomChange", + "clientId": "${common-client.client-id}", + "accessTokenValidity": "${common-client.access-token-validity}", + "refreshTokenValidity": "${common-client.refresh-token-validity}", + "authorizedGrantTypes": "${common-client.authorized-grant-types}" + } + # add a not-null constraint to the expiration columns + - addNotNullConstraint: + tableName: oauth_access_token + columnName: expiration + - addNotNullConstraint: + tableName: oauth_refresh_token + columnName: expiration diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.2.6.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.2.6.yaml new file mode 100644 index 0000000000..5afab9bacc --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.2.6.yaml @@ -0,0 +1,25 @@ +databaseChangeLog: + - changeSet: + id: drop-image-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: image + changes: + - dropTable: + schemaName: public + tableName: image + - changeSet: + id: drop-face-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: face + changes: + - dropTable: + schemaName: public + tableName: face diff --git a/java/api/src/test/resources/db/changelog/db.changelog-0.2.7.yaml b/java/api/src/test/resources/db/changelog/db.changelog-0.2.7.yaml new file mode 100644 index 0000000000..840f0ac15b --- /dev/null +++ b/java/api/src/test/resources/db/changelog/db.changelog-0.2.7.yaml @@ -0,0 +1,144 @@ +databaseChangeLog: + - changeSet: + id: drop-qrtz_blob_triggers_table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_blob_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_blob_triggers + - changeSet: + id: drop-qrtz_calendars-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_calendars + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_calendars + - changeSet: + id: drop-qrtz_cron_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_cron_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_cron_triggers + - changeSet: + id: drop-qrtz_fired_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_fired_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_fired_triggers + - changeSet: + id: drop-qrtz_job_details-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_job_details + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_job_details + - changeSet: + id: drop-qrtz_locks-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_locks + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_locks + - changeSet: + id: drop-qrtz_paused_trigger_grps-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_paused_trigger_grps + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_paused_trigger_grps + - changeSet: + id: drop-qrtz_scheduler_state-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_scheduler_state + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_scheduler_state + - changeSet: + id: drop-qrtz_simple_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_simple_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_simple_triggers + - changeSet: + id: drop-qrtz_simprop_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_simprop_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_simprop_triggers + - changeSet: + id: drop-qrtz_triggers-table + author: Volodymyr Bushko + preConditions: + - onFail: MARK_RAN + - tableExists: + schemaName: public + tableName: qrtz_triggers + changes: + - dropTable: + cascadeConstraints: true + schemaName: public + tableName: qrtz_triggers diff --git a/java/api/src/test/resources/db/changelog/db.changelog-master.yaml b/java/api/src/test/resources/db/changelog/db.changelog-master.yaml index b3ae37e3bb..d7dad0ef30 100644 --- a/java/api/src/test/resources/db/changelog/db.changelog-master.yaml +++ b/java/api/src/test/resources/db/changelog/db.changelog-master.yaml @@ -32,4 +32,24 @@ databaseChangeLog: - include: file: db/changelog/db.changelog-0.1.6.yaml - include: - file: db/changelog/db.changelog-0.1.7.yaml \ No newline at end of file + file: db/changelog/db.changelog-0.1.7.yaml + - include: + file: db/changelog/db.changelog-0.1.8.yaml + - include: + file: db/changelog/db.changelog-0.1.9.yaml + - include: + file: db/changelog/db.changelog-0.2.0.yaml + - include: + file: db/changelog/db.changelog-0.2.1.yaml + - include: + file: db/changelog/db.changelog-0.2.2.yaml + - include: + file: db/changelog/db.changelog-0.2.3.yaml + - include: + file: db/changelog/db.changelog-0.2.4.yaml + - include: + file: db/changelog/db.changelog-0.2.5.yaml + - include: + file: db/changelog/db.changelog-0.2.6.yaml + - include: + file: db/changelog/db.changelog-0.2.7.yaml diff --git a/java/common/pom.xml b/java/common/pom.xml index 27d4e1fa22..6917376fe1 100644 --- a/java/common/pom.xml +++ b/java/common/pom.xml @@ -54,13 +54,10 @@ org.springframework.cloud spring-cloud-starter-openfeign - - org.springframework.boot - spring-boot-starter-quartz - io.github.openfeign feign-jackson - \ No newline at end of file + + diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/annotation/CollectStatistics.java b/java/common/src/main/java/com/exadel/frs/commonservice/annotation/CollectStatistics.java index 5a899a139e..598cb14287 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/annotation/CollectStatistics.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/annotation/CollectStatistics.java @@ -15,4 +15,4 @@ public @interface CollectStatistics { StatisticsType type(); -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/aspect/StatisticsCollectionAspect.java b/java/common/src/main/java/com/exadel/frs/commonservice/aspect/StatisticsCollectionAspect.java index 41e4540519..0bee5d1f8d 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/aspect/StatisticsCollectionAspect.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/aspect/StatisticsCollectionAspect.java @@ -5,7 +5,6 @@ import com.exadel.frs.commonservice.entity.User; import com.exadel.frs.commonservice.enums.GlobalRole; import com.exadel.frs.commonservice.enums.StatisticsType; -import com.exadel.frs.commonservice.exception.ApperyServiceException; import com.exadel.frs.commonservice.repository.InstallInfoRepository; import com.exadel.frs.commonservice.repository.UserRepository; import com.exadel.frs.commonservice.system.feign.ApperyStatisticsClient; @@ -72,7 +71,7 @@ public void afterMethodInvocation(JoinPoint joinPoint, Object result) { new StatisticsGeneralEntity(getInstallGuid(), statisticsType) ); } catch (FeignException exception) { - throw new ApperyServiceException(); + log.info(exception.getMessage()); } } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/dto/ExecutionTimeDto.java b/java/common/src/main/java/com/exadel/frs/commonservice/dto/ExecutionTimeDto.java index 99daa448c3..2ea0a89cff 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/dto/ExecutionTimeDto.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/dto/ExecutionTimeDto.java @@ -14,6 +14,8 @@ public class ExecutionTimeDto { private Double age; private Double gender; + private Double pose; private Double detector; private Double calculator; -} \ No newline at end of file + private Double mask; +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacePredictionResultDto.java b/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacePredictionResultDto.java deleted file mode 100644 index 5c89c74ca4..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacePredictionResultDto.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import java.util.List; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(callSuper = true) -public class FacePredictionResultDto extends FindFacesResultDto { - - List faces; - List> landmarks; -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FaceResponseDto.java b/java/common/src/main/java/com/exadel/frs/commonservice/dto/FaceResponseDto.java deleted file mode 100644 index dc417cd770..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FaceResponseDto.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.dto; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class FaceResponseDto { - - private String image_id; - private String subject; -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacesDetectionResponseDto.java b/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacesDetectionResponseDto.java deleted file mode 100644 index ec12cb9b54..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacesDetectionResponseDto.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -package com.exadel.frs.commonservice.dto; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class FacesDetectionResponseDto { - - @JsonProperty(value = "plugins_versions") - private PluginsVersionsDto pluginsVersions; - private List result; -} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacesPoseDto.java b/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacesPoseDto.java new file mode 100644 index 0000000000..ec508fbe26 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/dto/FacesPoseDto.java @@ -0,0 +1,13 @@ +package com.exadel.frs.commonservice.dto; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class FacesPoseDto { + + private Double pitch; + private Double roll; + private Double yaw; +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FindFacesResultDto.java b/java/common/src/main/java/com/exadel/frs/commonservice/dto/FindFacesResultDto.java index da8d3a6bf5..e7f228445d 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FindFacesResultDto.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/dto/FindFacesResultDto.java @@ -15,17 +15,15 @@ */ package com.exadel.frs.commonservice.dto; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; - -import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; - @Data @Builder @NoArgsConstructor @@ -35,10 +33,11 @@ public class FindFacesResultDto { private FacesAgeDto age; private FacesGenderDto gender; + private FacesPoseDto pose; private Double[] embedding; private FacesBox box; @JsonProperty(value = "execution_time") private ExecutionTimeDto executionTime; private List> landmarks; - private FacesMaskDto maskDto; + private FacesMaskDto mask; } diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/dto/PluginsVersionsDto.java b/java/common/src/main/java/com/exadel/frs/commonservice/dto/PluginsVersionsDto.java index a329ab1605..0eda8f76d0 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/dto/PluginsVersionsDto.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/dto/PluginsVersionsDto.java @@ -32,6 +32,8 @@ public class PluginsVersionsDto { private String age; private String gender; + private String pose; private String detector; private String calculator; + private String mask; } diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/App.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/App.java index 642db4b45c..17515fff37 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/App.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/App.java @@ -49,11 +49,6 @@ public class App { @OneToMany(mappedBy = "app", cascade = CascadeType.ALL, orphanRemoval = true) private List userAppRoles = new ArrayList<>(); - @ToString.Exclude - @Builder.Default - @OneToMany(mappedBy = "app", cascade = CascadeType.ALL, orphanRemoval = true) - private List appModelAccess = new ArrayList<>(); - @ToString.Exclude @Builder.Default @OneToMany(mappedBy = "app", cascade = CascadeType.ALL, orphanRemoval = true) @@ -90,4 +85,4 @@ public void deleteUserAppRole(final String userGuid) { userAppRoles.remove(userAppRole); } } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/AppModel.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/AppModel.java deleted file mode 100644 index 575a9a65cf..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/AppModel.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.entity; - -import com.exadel.frs.commonservice.enums.AppModelAccess; -import com.exadel.frs.commonservice.helpers.ModelAccessTypeConverter; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import javax.persistence.*; - -@Entity -@Table(name = "app_model", schema = "public") -@Data -@NoArgsConstructor -@EqualsAndHashCode(of = {"app", "model"}) -public class AppModel { - - public AppModel(AppModel appModel){ - this.id = appModel.id; - this.app = appModel.app; - this.model = appModel.model; - this.accessType = appModel.accessType; - } - - @EmbeddedId - private AppModelId id; - - @ManyToOne(fetch = FetchType.LAZY, optional = false) - @MapsId("appId") - private App app; - - @ManyToOne(fetch = FetchType.LAZY, optional = false) - @MapsId("modelId") - private Model model; - - @Convert(converter = ModelAccessTypeConverter.class) - @Column(name = "access_type") - private AppModelAccess accessType; - - public AppModel(App app, Model model, AppModelAccess accessType) { - this.app = app; - this.model = model; - this.accessType = accessType; - this.id = new AppModelId(app.getId(), model.getId()); - } -} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/AppModelId.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/AppModelId.java deleted file mode 100644 index a88527e86a..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/AppModelId.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.entity; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import javax.persistence.Embeddable; -import java.io.Serializable; - -@Embeddable -@Data -@NoArgsConstructor -@AllArgsConstructor -public class AppModelId implements Serializable { - - private Long appId; - private Long modelId; -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/EmbeddingProjection.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/EmbeddingProjection.java deleted file mode 100644 index 36ef7927f3..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/EmbeddingProjection.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.exadel.frs.commonservice.entity; - -import lombok.Value; - -import java.util.UUID; - -@Value -public class EmbeddingProjection { - - UUID embeddingId; - String subjectName; - - public static EmbeddingProjection from(Embedding embedding) { - return new EmbeddingProjection( - embedding.getId(), - embedding.getSubject().getSubjectName() - ); - } - - public EmbeddingProjection withNewSubjectName(String newSubjectName) { - return new EmbeddingProjection( - this.getEmbeddingId(), - newSubjectName - ); - } -} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/Img.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/Img.java index 20763b5204..d02d9b9d5c 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/Img.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/Img.java @@ -20,4 +20,4 @@ public class Img { @Column(name = "content") private byte[] content; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/InstallInfo.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/InstallInfo.java index c083ad50a5..cd30542e7c 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/InstallInfo.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/InstallInfo.java @@ -15,4 +15,4 @@ public class InstallInfo { @Id @Column(name = "install_guid") private String installGuid; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/Model.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/Model.java index 5459490e3a..c7521054ec 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/Model.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/Model.java @@ -16,17 +16,30 @@ package com.exadel.frs.commonservice.entity; -import com.exadel.frs.commonservice.enums.AppModelAccess; +import static java.util.UUID.randomUUID; import com.exadel.frs.commonservice.enums.ModelType; import com.exadel.frs.commonservice.helpers.ModelTypeConverter; -import lombok.*; - -import javax.persistence.*; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; -import java.util.Optional; - -import static java.util.UUID.randomUUID; +import javax.persistence.CascadeType; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.ManyToOne; +import javax.persistence.OneToMany; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; @Entity @Table(schema = "public") @@ -43,8 +56,8 @@ public Model(Model model) { this.guid = randomUUID().toString(); this.apiKey = randomUUID().toString(); this.app = model.getApp(); - this.appModelAccess = model.appModelAccess; this.type = model.type; + this.createdDate = LocalDateTime.now(); } @Id @@ -64,17 +77,8 @@ public Model(Model model) { @ToString.Exclude @Builder.Default @OneToMany(mappedBy = "model", cascade = CascadeType.ALL, orphanRemoval = true) - private List appModelAccess = new ArrayList<>(); + private List modelStatistics = new ArrayList<>(); - public void addAppModelAccess(App app, AppModelAccess access) { - AppModel appModel = new AppModel(app, this, access); - appModelAccess.add(appModel); - app.getAppModelAccess().add(appModel); - } - - public Optional getAppModel(String appGuid) { - return appModelAccess.stream() - .filter(appModel -> appModel.getApp().getGuid().equals(appGuid)) - .findFirst(); - } -} \ No newline at end of file + @Column(name = "created_date") + private LocalDateTime createdDate; +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelShareRequest.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelShareRequest.java deleted file mode 100644 index 91f71a16a8..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelShareRequest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.entity; - -import lombok.*; -import org.hibernate.annotations.CreationTimestamp; - -import javax.persistence.*; -import java.time.LocalDateTime; - -@Entity -@Table(schema = "public") -@Data -@NoArgsConstructor -@AllArgsConstructor -@Builder -@EqualsAndHashCode(of = {"app", "requestTime"}) -public class ModelShareRequest { - - @EmbeddedId - private ModelShareRequestId id; - - @ManyToOne(fetch = FetchType.LAZY, optional = false) - @MapsId("appId") - private App app; - - @CreationTimestamp - @Column(name = "request_time") - private LocalDateTime requestTime; -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelShareRequestId.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelShareRequestId.java deleted file mode 100644 index 3d864eb11b..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelShareRequestId.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.entity; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import javax.persistence.Embeddable; -import java.io.Serializable; -import java.util.UUID; - -@Embeddable -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ModelShareRequestId implements Serializable { - - private Long appId; - private UUID requestId; -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelStatistic.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelStatistic.java new file mode 100644 index 0000000000..10fd5aae28 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelStatistic.java @@ -0,0 +1,44 @@ +package com.exadel.frs.commonservice.entity; + +import static javax.persistence.GenerationType.SEQUENCE; +import java.time.LocalDateTime; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Entity +@Table(name = "model_statistic", schema = "public") +@Getter +@Setter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ModelStatistic { + + @Id + @Column(name = "id") + @GeneratedValue(strategy = SEQUENCE, generator = "model_statistic_id_seq") + @SequenceGenerator(name = "model_statistic_id_seq", sequenceName = "model_statistic_id_seq", allocationSize = 1) + private Long id; + + @Column(name = "request_count") + private Integer requestCount; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "model_id", referencedColumnName = "id") + private Model model; + + @Column(name = "created_date") + private LocalDateTime createdDate; +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelSubjectProjection.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelSubjectProjection.java deleted file mode 100644 index d215ba3861..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/ModelSubjectProjection.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.exadel.frs.commonservice.entity; - -import lombok.Value; - -@Value -public class ModelSubjectProjection { - - String guid; - Long subjectCount; -} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/ResetPasswordToken.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/ResetPasswordToken.java new file mode 100644 index 0000000000..4d5c8a8f19 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/ResetPasswordToken.java @@ -0,0 +1,38 @@ +package com.exadel.frs.commonservice.entity; + +import java.time.LocalDateTime; +import java.util.UUID; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.OneToOne; +import javax.persistence.Table; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Entity +@Table(name = "reset_password_token", schema = "public") +@Getter +@Setter +@NoArgsConstructor +public class ResetPasswordToken { + + @Id + @GeneratedValue + private UUID token; + + @Column(name = "expires_in") + private LocalDateTime expiresIn; + + @OneToOne + @JoinColumn(name = "user_email", referencedColumnName = "email") + private User user; + + public ResetPasswordToken(final LocalDateTime expiresIn, final User user) { + this.expiresIn = expiresIn; + this.user = user; + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/TableLock.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/TableLock.java new file mode 100644 index 0000000000..84a8990dcb --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/TableLock.java @@ -0,0 +1,30 @@ +package com.exadel.frs.commonservice.entity; + +import com.exadel.frs.commonservice.enums.TableLockName; +import java.util.UUID; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Entity +@Table(name = "table_lock") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class TableLock { + + @Id + private UUID id; + + @Enumerated(EnumType.STRING) + @Column(name = "lock_name") + private TableLockName lockName; +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/User.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/User.java index b5df2580ba..58a3cdf08c 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/User.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/User.java @@ -82,4 +82,4 @@ public List getAuthorities() { public String getUsername() { return email; } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/entity/UserAppRoleId.java b/java/common/src/main/java/com/exadel/frs/commonservice/entity/UserAppRoleId.java index 37e94ac183..54e509cf71 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/entity/UserAppRoleId.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/entity/UserAppRoleId.java @@ -31,4 +31,4 @@ public class UserAppRoleId implements Serializable { private Long userId; private Long appId; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/enums/AppStatus.java b/java/common/src/main/java/com/exadel/frs/commonservice/enums/AppStatus.java new file mode 100644 index 0000000000..e740999223 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/enums/AppStatus.java @@ -0,0 +1,6 @@ +package com.exadel.frs.commonservice.enums; + +public enum AppStatus { + OK, + NOT_READY +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/enums/GlobalRole.java b/java/common/src/main/java/com/exadel/frs/commonservice/enums/GlobalRole.java index b83e293898..31b1ba77cb 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/enums/GlobalRole.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/enums/GlobalRole.java @@ -32,4 +32,4 @@ public enum GlobalRole implements EnumCode { @Getter @Setter private String code; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/enums/Replacer.java b/java/common/src/main/java/com/exadel/frs/commonservice/enums/Replacer.java index e691bd3bbf..6120a2a4a7 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/enums/Replacer.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/enums/Replacer.java @@ -34,4 +34,4 @@ public static Replacer from(final String text) { return Replacer.valueOf(text.toUpperCase()); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/enums/StatisticsType.java b/java/common/src/main/java/com/exadel/frs/commonservice/enums/StatisticsType.java index b9792f4e2e..679fd5b113 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/enums/StatisticsType.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/enums/StatisticsType.java @@ -18,4 +18,4 @@ public enum StatisticsType { @Getter @Setter private String code; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/enums/TableLockName.java b/java/common/src/main/java/com/exadel/frs/commonservice/enums/TableLockName.java new file mode 100644 index 0000000000..1372d28ade --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/enums/TableLockName.java @@ -0,0 +1,6 @@ +package com.exadel.frs.commonservice.enums; + +public enum TableLockName { + + MODEL_STATISTIC_LOCK +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/enums/ValidationResult.java b/java/common/src/main/java/com/exadel/frs/commonservice/enums/ValidationResult.java index 7d4294bc36..dc6e5b6bce 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/enums/ValidationResult.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/enums/ValidationResult.java @@ -20,4 +20,4 @@ public enum ValidationResult { OK, FORBIDDEN -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/ApperyServiceException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/ApperyServiceException.java index 6fe02eded5..590f3a4f35 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/ApperyServiceException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/ApperyServiceException.java @@ -9,4 +9,4 @@ public class ApperyServiceException extends BasicException { public ApperyServiceException() { super(APPERY_SERVICE_EXCEPTION, MESSAGE); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/BasicException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/BasicException.java index a4e920e12f..756c84987e 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/BasicException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/BasicException.java @@ -6,6 +6,10 @@ @Getter public class BasicException extends RuntimeException { + public enum LogLevel { + DEBUG, ERROR + } + private final HttpExceptionCode exceptionCode; private final String message; @@ -14,4 +18,8 @@ public BasicException(final HttpExceptionCode exceptionCode, final String messag this.exceptionCode = exceptionCode; this.message = message; } -} \ No newline at end of file + + public LogLevel getLogLevel() { + return LogLevel.ERROR; + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/DemoNotAvailableException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/DemoNotAvailableException.java index 3fef951ea6..aec6dbe7c6 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/DemoNotAvailableException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/DemoNotAvailableException.java @@ -30,4 +30,4 @@ public DemoNotAvailableException() { public synchronized Throwable fillInStackTrace() { return this; } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/EmptyRequiredFieldException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/EmptyRequiredFieldException.java index 3650ad366e..172e9ef18f 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/EmptyRequiredFieldException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/EmptyRequiredFieldException.java @@ -26,4 +26,4 @@ public class EmptyRequiredFieldException extends BasicException { public EmptyRequiredFieldException(final String fieldName) { super(EMPTY_REQUIRED_FIELD, format(MESSAGE, fieldName)); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/FileExtensionException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/FileExtensionException.java index 33c654e304..becea21141 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/FileExtensionException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/FileExtensionException.java @@ -26,4 +26,4 @@ public class FileExtensionException extends BasicException { public FileExtensionException(final String fileName) { super(UNAVAILABLE_FILE_EXTENSION, format(MESSAGE, fileName)); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/ImageNotFoundException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/ImageNotFoundException.java deleted file mode 100644 index c6bc328217..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/ImageNotFoundException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.exception; - -import static com.exadel.frs.commonservice.handler.CommonExceptionCode.IMAGE_NOT_FOUND; -import static java.lang.String.format; - -public class ImageNotFoundException extends BasicException { - - private static final String MESSAGE = "Image %s not found"; - - public ImageNotFoundException(String imageId) { - super(IMAGE_NOT_FOUND, format(MESSAGE, imageId)); - } - -} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectAccessTypeException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectAccessTypeException.java deleted file mode 100644 index d40660cb4a..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectAccessTypeException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.exception; - -import static com.exadel.frs.commonservice.handler.CommonExceptionCode.INCORRECT_ACCESS_TYPE; -import static java.lang.String.format; - -public class IncorrectAccessTypeException extends BasicException { - - public static final String ACCESS_TYPE_NOT_EXISTS_MESSAGE = "Access type %s does not exists"; - - public IncorrectAccessTypeException(final String accessType) { - super(INCORRECT_ACCESS_TYPE, format(ACCESS_TYPE_NOT_EXISTS_MESSAGE, accessType)); - } -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectAppRoleException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectAppRoleException.java index 554ae5a44c..da218b177a 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectAppRoleException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectAppRoleException.java @@ -26,4 +26,4 @@ public class IncorrectAppRoleException extends BasicException { public IncorrectAppRoleException(final String appRole) { super(INCORRECT_APP_ROLE, format(APP_ROLE_NOT_EXISTS_MESSAGE, appRole)); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectGlobalRoleException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectGlobalRoleException.java index 5363f22675..63d9adb03d 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectGlobalRoleException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectGlobalRoleException.java @@ -26,4 +26,4 @@ public class IncorrectGlobalRoleException extends BasicException { public IncorrectGlobalRoleException(final String globalRole) { super(INCORRECT_GLOBAL_ROLE, format(GLOBAL_ROLE_NOT_EXISTS_MESSAGE, globalRole)); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectImageIdException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectImageIdException.java new file mode 100644 index 0000000000..e327c660d8 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectImageIdException.java @@ -0,0 +1,10 @@ +package com.exadel.frs.commonservice.exception; + +import static com.exadel.frs.commonservice.handler.CommonExceptionCode.EMBEDDING_NOT_FOUND; + +public class IncorrectImageIdException extends BasicException { + + public IncorrectImageIdException() { + super(EMBEDDING_NOT_FOUND, "Image Id is incorrect"); + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectModelTypeException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectModelTypeException.java index 25452ea176..3fdbdb64c4 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectModelTypeException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectModelTypeException.java @@ -26,4 +26,4 @@ public class IncorrectModelTypeException extends BasicException { public IncorrectModelTypeException(final String modelType) { super(INCORRECT_MODEL_TYPE, format(MODEL_TYPE_NOT_EXISTS_MESSAGE, modelType)); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectPredictionCountException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectPredictionCountException.java index 7287b3f8c8..392228006a 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectPredictionCountException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/IncorrectPredictionCountException.java @@ -26,4 +26,4 @@ public class IncorrectPredictionCountException extends BasicException { public IncorrectPredictionCountException() { super(INCORRECT_MODEL_TYPE, INCORRECT_PREDICTION_MESSAGE); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/InvalidBase64Exception.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/InvalidBase64Exception.java index f9f66d6f6b..1e95b9c098 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/InvalidBase64Exception.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/InvalidBase64Exception.java @@ -25,4 +25,4 @@ public class InvalidBase64Exception extends BasicException { public InvalidBase64Exception() { super(UNAVAILABLE_FILE_EXTENSION, MESSAGE); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/PassedIncorrectArgumentException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/PassedIncorrectArgumentException.java deleted file mode 100644 index 7bed5f2a4c..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/PassedIncorrectArgumentException.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.exadel.frs.commonservice.exception; - -import static com.exadel.frs.commonservice.handler.CommonExceptionCode.MISSING_PATH_VARIABLE; -import static java.lang.String.format; - -public class PassedIncorrectArgumentException extends BasicException { - - public static final String INCORRECT_ARGUMENT_MESSAGE = "Passed argument is incorrect: %s"; - - public PassedIncorrectArgumentException(final String exceptionMessage) { - super(MISSING_PATH_VARIABLE, format(INCORRECT_ARGUMENT_MESSAGE, exceptionMessage)); - } -} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/PatternMatchException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/PatternMatchException.java new file mode 100644 index 0000000000..5627e04460 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/PatternMatchException.java @@ -0,0 +1,10 @@ +package com.exadel.frs.commonservice.exception; + +import static com.exadel.frs.commonservice.handler.CommonExceptionCode.INCORRECT_ARGUMENT; + +public class PatternMatchException extends BasicException { + + public PatternMatchException(final String message) { + super(INCORRECT_ARGUMENT, message); + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/TooManyFacesException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/TooManyFacesException.java index 95b8305f0e..d5f18d5684 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/exception/TooManyFacesException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/TooManyFacesException.java @@ -25,4 +25,9 @@ public class TooManyFacesException extends BasicException { public TooManyFacesException() { super(TOO_MANY_FACES, MESSAGE); } -} \ No newline at end of file + + @Override + public LogLevel getLogLevel() { + return LogLevel.DEBUG; + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/exception/WrongEmbeddingCountException.java b/java/common/src/main/java/com/exadel/frs/commonservice/exception/WrongEmbeddingCountException.java new file mode 100644 index 0000000000..7b48021a98 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/exception/WrongEmbeddingCountException.java @@ -0,0 +1,17 @@ +package com.exadel.frs.commonservice.exception; + +import static com.exadel.frs.commonservice.handler.CommonExceptionCode.WRONG_EMBEDDING_COUNT; +import com.exadel.frs.commonservice.handler.HttpExceptionCode; + +public class WrongEmbeddingCountException extends BasicException { + + private static final String MESSAGE = "%d embeddings were expected, but %d were provided"; + + public WrongEmbeddingCountException(final int expectedCount, final int providedCount) { + super(WRONG_EMBEDDING_COUNT, String.format(MESSAGE, expectedCount, providedCount)); + } + + public WrongEmbeddingCountException(final HttpExceptionCode exceptionCode, final String message) { + super(exceptionCode, message); + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/handler/CommonExceptionCode.java b/java/common/src/main/java/com/exadel/frs/commonservice/handler/CommonExceptionCode.java index efb7632c68..f6f69f507a 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/handler/CommonExceptionCode.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/handler/CommonExceptionCode.java @@ -50,6 +50,7 @@ public enum CommonExceptionCode implements HttpExceptionCode { MISSING_REQUEST_PART(34, BAD_REQUEST), MISSING_PATH_VARIABLE(35, BAD_REQUEST), INCORRECT_ARGUMENT(36, BAD_REQUEST), + WRONG_EMBEDDING_COUNT(37, BAD_REQUEST), FACES_SERVICE_EXCEPTION(41, INTERNAL_SERVER_ERROR), SUBJECT_NOT_FOUND(42, NOT_FOUND), @@ -60,4 +61,4 @@ public enum CommonExceptionCode implements HttpExceptionCode { private final Integer code; private final HttpStatus httpStatus; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/handler/CrudExceptionCode.java b/java/common/src/main/java/com/exadel/frs/commonservice/handler/CrudExceptionCode.java index eb21ab34a7..1072f979f9 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/handler/CrudExceptionCode.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/handler/CrudExceptionCode.java @@ -52,9 +52,11 @@ public enum CrudExceptionCode implements HttpExceptionCode { APPERY_SERVICE_EXCEPTION(27, INTERNAL_SERVER_ERROR), INCORRECT_STATISTICS_ROLE(28, BAD_REQUEST), + MAIL_SERVER_EXCEPTION(29, INTERNAL_SERVER_ERROR), + INVALID_RESET_PASSWORD_TOKEN(30, BAD_REQUEST), UNDEFINED(0, BAD_REQUEST); private final Integer code; private final HttpStatus httpStatus; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/handler/ResponseExceptionHandler.java b/java/common/src/main/java/com/exadel/frs/commonservice/handler/ResponseExceptionHandler.java index 30dc7862a4..663bb1515d 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/handler/ResponseExceptionHandler.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/handler/ResponseExceptionHandler.java @@ -30,6 +30,7 @@ import com.exadel.frs.commonservice.exception.MissingPathVarException; import com.exadel.frs.commonservice.exception.MissingRequestParamException; import com.exadel.frs.commonservice.exception.MissingRequestPartException; +import com.exadel.frs.commonservice.exception.PatternMatchException; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.http.HttpHeaders; @@ -53,7 +54,10 @@ public class ResponseExceptionHandler extends ResponseEntityExceptionHandler { @ExceptionHandler(BasicException.class) public ResponseEntity handleDefinedExceptions(final BasicException ex) { - log.error("Defined exception occurred", ex); + switch (ex.getLogLevel()) { + case ERROR -> log.error("Defined exception occurred", ex); + case DEBUG -> log.debug("Defined exception occurred", ex); + } return ResponseEntity .status(ex.getExceptionCode().getHttpStatus()) @@ -181,21 +185,12 @@ private static BasicException getException(final FieldError fieldError) { return new BasicException(UNDEFINED, ""); } - switch (code) { - case "NotBlank": - case "ValidEnum": - basicException = new ConstraintViolationException(fieldError.getDefaultMessage()); - break; - case "NotNull": - case "NotEmpty": - basicException = new EmptyRequiredFieldException(fieldError.getField()); - break; - case "Size": - basicException = new ConstraintViolationException(fieldError.getField(), fieldError.getDefaultMessage()); - break; - default: - basicException = new BasicException(UNDEFINED, ""); - } + basicException = switch (code) { + case "NotBlank", "ValidEnum", "Size" -> new ConstraintViolationException(fieldError.getDefaultMessage()); + case "NotNull", "NotEmpty" -> new EmptyRequiredFieldException(fieldError.getField()); + case "Pattern" -> new PatternMatchException(fieldError.getDefaultMessage()); + default -> new BasicException(UNDEFINED, ""); + }; return basicException; } @@ -221,4 +216,4 @@ private static ExceptionResponseDto buildBody(Integer code, String message) { .message(message) .build(); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/helpers/AppRoleConverter.java b/java/common/src/main/java/com/exadel/frs/commonservice/helpers/AppRoleConverter.java index 0e27a8e877..04dca6194b 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/helpers/AppRoleConverter.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/helpers/AppRoleConverter.java @@ -28,4 +28,4 @@ public class AppRoleConverter extends EnumCodeConverter { public AppRole convertToEntityAttribute(String code) { return super.convertToEntityAttribute(code, AppRole.values(), new IncorrectAppRoleException(code)); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/helpers/GlobalRoleConverter.java b/java/common/src/main/java/com/exadel/frs/commonservice/helpers/GlobalRoleConverter.java index 953823255d..48c0cd05ff 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/helpers/GlobalRoleConverter.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/helpers/GlobalRoleConverter.java @@ -28,4 +28,4 @@ public class GlobalRoleConverter extends EnumCodeConverter { public GlobalRole convertToEntityAttribute(String code) { return super.convertToEntityAttribute(code, GlobalRole.values(), new IncorrectGlobalRoleException(code)); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/helpers/ModelAccessTypeConverter.java b/java/common/src/main/java/com/exadel/frs/commonservice/helpers/ModelAccessTypeConverter.java deleted file mode 100644 index 836b1f74ef..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/helpers/ModelAccessTypeConverter.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.helpers; - -import com.exadel.frs.commonservice.enums.AppModelAccess; -import com.exadel.frs.commonservice.exception.IncorrectAccessTypeException; - -import javax.persistence.Converter; - -@Converter(autoApply = true) -public class ModelAccessTypeConverter extends EnumCodeConverter { - - @Override - public AppModelAccess convertToEntityAttribute(String code) { - return super.convertToEntityAttribute(code, AppModelAccess.values(), new IncorrectAccessTypeException(code)); - } -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/helpers/ModelTypeConverter.java b/java/common/src/main/java/com/exadel/frs/commonservice/helpers/ModelTypeConverter.java index d2c2edae1a..459d4bfaa7 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/helpers/ModelTypeConverter.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/helpers/ModelTypeConverter.java @@ -28,4 +28,4 @@ public class ModelTypeConverter extends EnumCodeConverter { public ModelType convertToEntityAttribute(String code) { return super.convertToEntityAttribute(code, ModelType.values(), new IncorrectModelTypeException(code)); } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/projection/EmbeddingProjection.java b/java/common/src/main/java/com/exadel/frs/commonservice/projection/EmbeddingProjection.java new file mode 100644 index 0000000000..65f8b7eb65 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/projection/EmbeddingProjection.java @@ -0,0 +1,28 @@ +package com.exadel.frs.commonservice.projection; + +import com.exadel.frs.commonservice.entity.Embedding; +import java.util.UUID; + +public record EmbeddingProjection(UUID embeddingId, String subjectName) { + + public static EmbeddingProjection from(Embedding embedding) { + return new EmbeddingProjection( + embedding.getId(), + embedding.getSubject().getSubjectName() + ); + } + + public static EmbeddingProjection from(EnhancedEmbeddingProjection projection) { + return new EmbeddingProjection( + projection.embeddingId(), + projection.subjectName() + ); + } + + public EmbeddingProjection withNewSubjectName(String newSubjectName) { + return new EmbeddingProjection( + this.embeddingId(), + newSubjectName + ); + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/projection/EnhancedEmbeddingProjection.java b/java/common/src/main/java/com/exadel/frs/commonservice/projection/EnhancedEmbeddingProjection.java new file mode 100644 index 0000000000..0bb47521dd --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/projection/EnhancedEmbeddingProjection.java @@ -0,0 +1,10 @@ +package com.exadel.frs.commonservice.projection; + +import java.util.UUID; + +/** + * @param embeddingData embedding column of embedding table + */ +public record EnhancedEmbeddingProjection(UUID embeddingId, double[] embeddingData, String subjectName) { + +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelProjection.java b/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelProjection.java new file mode 100644 index 0000000000..448bba2710 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelProjection.java @@ -0,0 +1,14 @@ +package com.exadel.frs.commonservice.projection; + +import com.exadel.frs.commonservice.enums.ModelType; +import java.time.LocalDateTime; + +public record ModelProjection( + + String guid, + String name, + String apiKey, + ModelType type, + LocalDateTime createdDate) { + +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelStatisticProjection.java b/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelStatisticProjection.java new file mode 100644 index 0000000000..7f7e3fc0e9 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelStatisticProjection.java @@ -0,0 +1,7 @@ +package com.exadel.frs.commonservice.projection; + +import java.util.Date; + +public record ModelStatisticProjection(long requestCount, Date createdDate) { + +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelSubjectProjection.java b/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelSubjectProjection.java new file mode 100644 index 0000000000..0e454b2f74 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/projection/ModelSubjectProjection.java @@ -0,0 +1,5 @@ +package com.exadel.frs.commonservice.projection; + +public record ModelSubjectProjection(String guid, Long subjectCount) { + +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/repository/EmbeddingRepository.java b/java/common/src/main/java/com/exadel/frs/commonservice/repository/EmbeddingRepository.java index 8ecf4175e4..827491fe06 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/repository/EmbeddingRepository.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/repository/EmbeddingRepository.java @@ -1,9 +1,12 @@ package com.exadel.frs.commonservice.repository; import com.exadel.frs.commonservice.entity.Embedding; -import com.exadel.frs.commonservice.entity.EmbeddingProjection; -import com.exadel.frs.commonservice.entity.Img; +import com.exadel.frs.commonservice.projection.EmbeddingProjection; +import com.exadel.frs.commonservice.projection.EnhancedEmbeddingProjection; import com.exadel.frs.commonservice.entity.Subject; +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.EntityGraph; @@ -12,15 +15,20 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import java.util.List; -import java.util.UUID; -import java.util.stream.Stream; - public interface EmbeddingRepository extends JpaRepository { // Note: consumer should consume in transaction - @EntityGraph("embedding-with-subject") - Stream findBySubjectApiKey(String apiKey); + @Query(""" + select + new com.exadel.frs.commonservice.projection.EnhancedEmbeddingProjection(e.id, e.embedding, s.subjectName) + from + Embedding e + left join + e.subject s + where + s.apiKey = :apiKey + """) + Stream findBySubjectApiKey(@Param("apiKey") String apiKey); @EntityGraph("embedding-with-subject") List findBySubjectId(UUID subjectId); @@ -46,43 +54,54 @@ int updateEmbedding(@Param("embeddingId") UUID embeddingId, @Query("update Embedding e set e.subject = :toSubject where e.subject = :fromSubject") int reassignEmbeddings(@Param("fromSubject") Subject fromSubject, @Param("toSubject") Subject toSubject); - @Query("select " + - " new com.exadel.frs.commonservice.entity.EmbeddingProjection(e.id, e.subject.subjectName)" + - " from " + - " Embedding e " + - " where " + - " e.subject.apiKey = :apiKey") + @Query(""" + select + new com.exadel.frs.commonservice.projection.EmbeddingProjection(e.id, e.subject.subjectName) + from + Embedding e + where + e.subject.apiKey = :apiKey + """) Page findBySubjectApiKey(String apiKey, Pageable pageable); + @Query(""" + select + new com.exadel.frs.commonservice.projection.EmbeddingProjection(e.id, e.subject.subjectName) + from + Embedding e + where + e.subject.apiKey = :apiKey + and + (cast(:subjectName as string) is null or e.subject.subjectName = :subjectName) + """) + Page findBySubjectApiKeyAndSubjectName(String apiKey, String subjectName, Pageable pageable); + @Query("select distinct(e.calculator) from Embedding e") List getUniqueCalculators(); - @Query("select " + - " count(e) " + - " from " + - " Embedding e " + - " where " + - " e.subject.apiKey = :apiKey " + - " and e.calculator <> :calculator") + @Query(""" + select + count(e) + from + Embedding e + where + e.subject.apiKey = :apiKey + and + e.calculator <> :calculator + """) Long countBySubjectApiKeyAndCalculatorNotEq(@Param("apiKey") String apiKey, @Param("calculator") String calculator); - @Query("select " + - " count(e) " + - " from " + - " Embedding e " + - " where " + - " e.subject.apiKey <> :apiKey " + - " and e.calculator <> :calculator") + @Query(""" + select + count(e) + from + Embedding e + where + e.subject.apiKey <> :apiKey + and + e.calculator <> :calculator + """) Long countBySubjectApiKeyNotEqAndCalculatorNotEq(@Param("apiKey") String apiKey, @Param("calculator") String calculator); - - @EntityGraph("embedding-with-subject") - @Query(value = "select e from Embedding e inner join fetch e.subject s inner join fetch e.img where s.id = :subject_id") - List findEmbeddingsBySubjectId(@Param("subject_id") UUID subjectId); - - List findBySubjectIdIn( List subjectIds ); - - @EntityGraph("embedding-with-subject") - List findBySubject( Subject subject ); -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/repository/ImgRepository.java b/java/common/src/main/java/com/exadel/frs/commonservice/repository/ImgRepository.java index 8733f54b81..b84e5a4b84 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/repository/ImgRepository.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/repository/ImgRepository.java @@ -26,4 +26,7 @@ public interface ImgRepository extends PagingAndSortingRepository { @Query("select i from Img i join Embedding e on e.img.id = i.id where e.id = :embeddingId and e.subject.apiKey = :apiKey") Optional getImgByEmbeddingId(@Param("apiKey") String apiKey, @Param("embeddingId") UUID embeddingId); + + @Query("select count(i) from Img i join Embedding e on e.img = i.id join Subject s on e.subject.id = s.id where s.apiKey=:apiKey") + Long getImageCountByApiKey(@Param("apiKey") String apiKey); } diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/repository/InstallInfoRepository.java b/java/common/src/main/java/com/exadel/frs/commonservice/repository/InstallInfoRepository.java index 32f9b3ca36..2d890a7a27 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/repository/InstallInfoRepository.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/repository/InstallInfoRepository.java @@ -24,4 +24,4 @@ public interface InstallInfoRepository extends JpaRepository { InstallInfo findTopByOrderByInstallGuid(); -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/repository/ModelRepository.java b/java/common/src/main/java/com/exadel/frs/commonservice/repository/ModelRepository.java index 9ba941d87a..2b05afc92b 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/repository/ModelRepository.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/repository/ModelRepository.java @@ -17,34 +17,71 @@ package com.exadel.frs.commonservice.repository; import com.exadel.frs.commonservice.entity.Model; -import com.exadel.frs.commonservice.entity.ModelSubjectProjection; import com.exadel.frs.commonservice.enums.ModelType; +import com.exadel.frs.commonservice.projection.ModelProjection; +import com.exadel.frs.commonservice.projection.ModelSubjectProjection; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; -import java.util.List; -import java.util.Optional; - @Repository public interface ModelRepository extends JpaRepository { + Optional findByApiKeyAndType(String apiKey, ModelType type); - @Query("select distinct m " + - "from Model m " + - "left join AppModel am on m.id = am.id.modelId " + - "where am.id.appId = :appId OR m.app.id = :appId") - List findAllByAppId(Long appId); + Stream findAllByIdIn(Set ids); Optional findByGuid(String guid); - boolean existsByNameAndAppId(String name, Long appId); + @Query(""" + select + case when count(m) > 0 then TRUE else FALSE end + from + Model m + where + lower(m.name) = lower(:name) + and + m.app.id = :appId + """) + boolean existsByUniqueNameAndAppId(String name, Long appId); - @Query("SELECT " + - " new com.exadel.frs.commonservice.entity.ModelSubjectProjection(m.guid, count(s.id)) " + - " FROM " + - " Model m LEFT JOIN Subject s ON m.apiKey = s.apiKey " + - " GROUP BY " + - " m.guid") + @Query(""" + select + count(m) + from + Model m + where + lower(m.name) = lower(:name) + and + m.app.id = :appId + """) + int countByUniqueNameAndAppId(String name, Long appId); + + @Query(""" + select + new com.exadel.frs.commonservice.projection.ModelSubjectProjection(m.guid, count(s.id)) + from + Model m + left join + Subject s on m.apiKey = s.apiKey + group by + m.guid + """) List getModelSubjectsCount(); + + @Query(""" + select distinct + new com.exadel.frs.commonservice.projection.ModelProjection(m.guid, m.name, m.apiKey, m.type, m.createdDate) + from + Model m + left join + m.app a + where + a.id = :appId + """) + List findAllByAppId(Long appId); } diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/repository/ModelStatisticRepository.java b/java/common/src/main/java/com/exadel/frs/commonservice/repository/ModelStatisticRepository.java new file mode 100644 index 0000000000..fdc012752b --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/repository/ModelStatisticRepository.java @@ -0,0 +1,36 @@ +package com.exadel.frs.commonservice.repository; + +import com.exadel.frs.commonservice.entity.ModelStatistic; +import com.exadel.frs.commonservice.projection.ModelStatisticProjection; +import java.time.LocalDateTime; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +@Repository +public interface ModelStatisticRepository extends JpaRepository { + + Stream findAllByModelIdInAndCreatedDate(Set modelIds, LocalDateTime createdDate); + + @Query(""" + select + new com.exadel.frs.commonservice.projection.ModelStatisticProjection(sum(statistic.requestCount), cast(statistic.createdDate as date)) + from + ModelStatistic as statistic + join + statistic.model as model + where + model.guid = :modelGuid + and + cast(statistic.createdDate as date) between :startDate and :endDate + group by + cast(statistic.createdDate as date) + order by + cast(statistic.createdDate as date) desc + """) + List findAllSummarizedByDay(String modelGuid, Date startDate, Date endDate); +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/repository/SubjectRepository.java b/java/common/src/main/java/com/exadel/frs/commonservice/repository/SubjectRepository.java index b3e311a72f..7e7e8b0bb1 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/repository/SubjectRepository.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/repository/SubjectRepository.java @@ -23,4 +23,6 @@ public interface SubjectRepository extends PagingAndSortingRepository { + + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("select tl from TableLock tl where UPPER(tl.lockName) = UPPER(:#{#lockName?.toString()})") + TableLock lockByName(@Param("lockName") TableLockName lockName); +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/repository/UserRepository.java b/java/common/src/main/java/com/exadel/frs/commonservice/repository/UserRepository.java index 2ba9a8b62c..8ed3c31bc0 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/repository/UserRepository.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/repository/UserRepository.java @@ -43,9 +43,12 @@ public interface UserRepository extends JpaRepository { User findByGlobalRole(GlobalRole role); + @Query("select count(u) > 0 from User u where u.globalRole = 'O'") + boolean isOwnerPresent(); + int deleteByEnabledFalseAndRegTimeBefore(LocalDateTime time); Optional findByRegistrationToken(String token); void deleteByGuid(String userGuid); -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/scheduler/config/SpringQuartzSchedulerConfig.java b/java/common/src/main/java/com/exadel/frs/commonservice/scheduler/config/SpringQuartzSchedulerConfig.java deleted file mode 100644 index 91c7f85aa7..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/scheduler/config/SpringQuartzSchedulerConfig.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.exadel.frs.commonservice.scheduler.config; - -import com.exadel.frs.commonservice.scheduler.job.StatisticsJob; -import org.quartz.*; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.quartz.SchedulerFactoryBean; - -@Configuration -public class SpringQuartzSchedulerConfig { - - @Bean - JobDetail jobDetail() { - return JobBuilder.newJob(StatisticsJob.class) - .withIdentity("StatisticsJob") - .storeDurably() - .build(); - } - - @Bean - Trigger trigger() { - TriggerBuilder triggerBuilder = TriggerBuilder - .newTrigger() - .forJob(jobDetail()) - .withIdentity("Statistics trigger"); - triggerBuilder - .withSchedule(SimpleScheduleBuilder - .simpleSchedule() - .withIntervalInHours(24) - .repeatForever()) - .startNow(); - return triggerBuilder.build(); - } - - @Bean - public Scheduler scheduler(Trigger trigger, JobDetail job, SchedulerFactoryBean factory) throws SchedulerException { - Scheduler scheduler = factory.getScheduler(); - scheduler.start(); - return scheduler; - } -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/scheduler/job/StatisticsJob.java b/java/common/src/main/java/com/exadel/frs/commonservice/scheduler/job/StatisticsJob.java deleted file mode 100644 index 802ec639aa..0000000000 --- a/java/common/src/main/java/com/exadel/frs/commonservice/scheduler/job/StatisticsJob.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.exadel.frs.commonservice.scheduler.job; - -import com.exadel.frs.commonservice.entity.ModelSubjectProjection; -import com.exadel.frs.commonservice.entity.User; -import com.exadel.frs.commonservice.enums.GlobalRole; -import com.exadel.frs.commonservice.exception.ApperyServiceException; -import com.exadel.frs.commonservice.repository.InstallInfoRepository; -import com.exadel.frs.commonservice.repository.ModelRepository; -import com.exadel.frs.commonservice.repository.UserRepository; -import com.exadel.frs.commonservice.system.feign.ApperyStatisticsClient; -import com.exadel.frs.commonservice.system.feign.StatisticsFacesEntity; -import feign.FeignException; -import lombok.NoArgsConstructor; -import org.apache.commons.lang3.Range; -import org.apache.commons.lang3.StringUtils; -import org.quartz.JobExecutionContext; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.quartz.QuartzJobBean; -import org.springframework.stereotype.Component; - -import java.util.List; -import java.util.Objects; - -import static org.apache.commons.lang3.Range.between; - -@NoArgsConstructor -@Component -public class StatisticsJob extends QuartzJobBean { - - @Value("${app.feign.appery-io.api-key}") - private String statisticsApiKey; - private ApperyStatisticsClient apperyStatisticsClient; - private InstallInfoRepository installInfoRepository; - private ModelRepository modelRepository; - private UserRepository userRepository; - - private List ranges = List.of( - Range.between(1, 10), - Range.between(11, 50), - Range.between(51, 200), - Range.between(201, 500), - Range.between(501, 2000), - between(2001, 10000), - between(10001, 50000), - between(50001, 200000), - between(200001, 1000000) - ); - - @Autowired - public StatisticsJob(final ApperyStatisticsClient apperyStatisticsClient, final InstallInfoRepository installInfoRepository, - final ModelRepository modelRepository, - final UserRepository userRepository) { - this.apperyStatisticsClient = apperyStatisticsClient; - this.installInfoRepository = installInfoRepository; - this.modelRepository = modelRepository; - this.userRepository = userRepository; - } - - @Override - public void executeInternal(final JobExecutionContext context) { - if (StringUtils.isEmpty(statisticsApiKey)) { - return; - } - - User user = userRepository.findByGlobalRole(GlobalRole.OWNER); - - if (Objects.isNull(user)) { - return; - } - - if (!user.isAllowStatistics()) { - return; - } - - List projections = modelRepository.getModelSubjectsCount(); - String installGuid = installInfoRepository.findTopByOrderByInstallGuid().getInstallGuid(); - - try { - for (ModelSubjectProjection projection : projections) { - apperyStatisticsClient.create(statisticsApiKey, new StatisticsFacesEntity( - installGuid, projection.getGuid(), getSubjectsRange(projection.getSubjectCount()) - )); - } - } catch (FeignException exception) { - throw new ApperyServiceException(); - } - } - - private String getSubjectsRange(Long subjectCount) { - if (subjectCount == 0) { - return "0"; - } - - for (Range range : ranges) { - if (range.contains(subjectCount)) { - return range.getMinimum() + "-" + range.getMaximum(); - } - } - - return "1000001+"; - } -} \ No newline at end of file diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/config/FeignClientsConfig.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/config/FeignClientsConfig.java index a656829875..e31e8ef974 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/config/FeignClientsConfig.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/config/FeignClientsConfig.java @@ -17,29 +17,50 @@ package com.exadel.frs.commonservice.sdk.config; import static com.exadel.frs.commonservice.system.global.EnvironmentProperties.ServerType.PYTHON; +import static com.zaxxer.hikari.util.ClockSource.toMillis; +import static feign.Logger.Level.FULL; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import com.exadel.frs.commonservice.sdk.faces.feign.FacesFeignClient; import com.exadel.frs.commonservice.system.global.EnvironmentProperties; import feign.Feign; -import feign.Logger; +import feign.Request; +import feign.Retryer; import feign.form.spring.SpringFormEncoder; import feign.jackson.JacksonDecoder; import feign.jackson.JacksonEncoder; import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; +import org.springframework.context.annotation.Configuration; -@Component +@Configuration @RequiredArgsConstructor public class FeignClientsConfig { + @Value("${app.feign.faces.connect-timeout}") + private int facesConnectTimeout; + + @Value("${app.feign.faces.read-timeout}") + private int facesReadTimeout; + + @Value("${app.feign.faces.retryer.max-attempts}") + private int facesRetryerMaxAttempts; + private final EnvironmentProperties properties; @Bean - public FacesFeignClient getFacesClient() { + public FacesFeignClient facesFeignClient() { return Feign.builder() .encoder(new SpringFormEncoder(new JacksonEncoder())) .decoder(new JacksonDecoder()) - .logLevel(Logger.Level.FULL) + .logLevel(FULL) + .retryer(facesFeignRetryer()) + .options(new Request.Options(facesConnectTimeout, MILLISECONDS, facesReadTimeout, MILLISECONDS, true)) .target(FacesFeignClient.class, properties.getServers().get(PYTHON).getUrl()); } -} \ No newline at end of file + + @Bean + public Retryer facesFeignRetryer() { + return new Retryer.Default(100, toMillis(1), facesRetryerMaxAttempts); + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/FacesApiClient.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/FacesApiClient.java index cb4c44b9ac..1558cc5de1 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/FacesApiClient.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/FacesApiClient.java @@ -22,7 +22,8 @@ FindFacesResponse findFaces( MultipartFile photo, Integer faceLimit, Double thresholdC, - String facePlugins); + String facePlugins, + Boolean detectFaces); /** * Calls /find_faces_base64 endpoint of Faces API @@ -37,7 +38,8 @@ FindFacesResponse findFacesBase64( String imageAsBase64, Integer faceLimit, Double thresholdC, - String facePlugins); + String facePlugins, + Boolean detectFaces); /** * Calls /find_faces endpoint of Faces API with 'calculator' plugin always on @@ -46,7 +48,8 @@ FindFacesResponse findFacesWithCalculator( MultipartFile photo, Integer faceLimit, Double thresholdC, - String facePlugins); + String facePlugins, + Boolean detectFaces); /** * Calls /find_faces endpoint of Faces API with 'calculator' plugin always on @@ -55,7 +58,8 @@ FindFacesResponse findFacesBase64WithCalculator( String imageAsBase64, Integer faceLimit, Double thresholdC, - String facePlugins); + String facePlugins, + Boolean detectFaces); /** * Calls /status endpoint of Faces API diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/exception/NoFacesFoundException.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/exception/NoFacesFoundException.java index d64ab56d8a..5d3bec5b5c 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/exception/NoFacesFoundException.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/exception/NoFacesFoundException.java @@ -28,4 +28,9 @@ public class NoFacesFoundException extends BasicException { public NoFacesFoundException() { super(NO_FACES_FOUND, MESSAGE); } -} \ No newline at end of file + + @Override + public LogLevel getLogLevel() { + return LogLevel.DEBUG; + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/FacesFeignClient.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/FacesFeignClient.java index f75fbabcb5..eb4115f75a 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/FacesFeignClient.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/FacesFeignClient.java @@ -31,23 +31,30 @@ public interface FacesFeignClient { @Headers("Content-Type: multipart/form-data") FindFacesResponse findFaces( @Param(value = "file") - MultipartFile photo, + MultipartFile photo, @Param(value = "limit") - Integer faceLimit, + Integer faceLimit, @Param(value = "det_prob_threshold") - Double thresholdC, + Double thresholdC, @Param(value = "face_plugins") - String facePlugins); + String facePlugins, + @Param(value = "detect_faces") + Boolean detectFaces); - @RequestLine("POST /find_faces_base64?limit={limit}&det_prob_threshold={threshold}&face_plugins={plugins}") + @RequestLine("POST /find_faces_base64?limit={limit}&det_prob_threshold={threshold}&face_plugins={plugins}&detect_faces={detect_faces}") @Headers("Content-Type: " + MediaType.APPLICATION_JSON_VALUE) FindFacesResponse findFacesBase64( FindFacesRequest request, - @Param(value = "limit") Integer faceLimit, - @Param(value = "threshold") Double thresholdC, - @Param(value = "plugins") String facePlugins); + @Param(value = "limit") + Integer faceLimit, + @Param(value = "threshold") + Double thresholdC, + @Param(value = "plugins") + String facePlugins, + @Param(value = "detect_faces") + Boolean detectFaces); @RequestLine("GET /status") @Headers("Content-Type: multipart/form-data") FacesStatusResponse getStatus(); -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesAge.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesAge.java index 9d9c2be500..f2b333774f 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesAge.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesAge.java @@ -6,6 +6,7 @@ @Data @Accessors(chain = true) public class FacesAge { + private Double probability; private Integer high; private Integer low; diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesGender.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesGender.java index eb00f1f63c..9a4292d138 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesGender.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesGender.java @@ -6,6 +6,7 @@ @Data @Accessors(chain = true) public class FacesGender { + private Double probability; private String value; } diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesMask.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesMask.java index 70acc366f3..30969efc0b 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesMask.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesMask.java @@ -6,6 +6,7 @@ @Data @Accessors(chain = true) public class FacesMask { + private Double probability; private String value; } diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesPose.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesPose.java new file mode 100644 index 0000000000..670009f8d6 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesPose.java @@ -0,0 +1,13 @@ +package com.exadel.frs.commonservice.sdk.faces.feign.dto; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class FacesPose { + + private Double pitch; + private Double roll; + private Double yaw; +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesStatusResponse.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesStatusResponse.java index fd9cbad089..d7d6067609 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesStatusResponse.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FacesStatusResponse.java @@ -41,4 +41,4 @@ public class FacesStatusResponse { @JsonProperty(value = "available_plugins") private Map availablePlugins; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FindFacesRequest.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FindFacesRequest.java index 672675603d..57257951df 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FindFacesRequest.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FindFacesRequest.java @@ -1,6 +1,5 @@ package com.exadel.frs.commonservice.sdk.faces.feign.dto; - import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -8,6 +7,7 @@ @Data @RequiredArgsConstructor public class FindFacesRequest { + @JsonProperty("file") private final String imageAsBase64; } diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FindFacesResult.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FindFacesResult.java index bf7a1cd564..682df4bba6 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FindFacesResult.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/FindFacesResult.java @@ -36,6 +36,7 @@ public class FindFacesResult { private FacesAge age; private FacesGender gender; + private FacesPose pose; private Double[] embedding; private FacesBox box; @JsonProperty(value = "execution_time") diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/PluginsVersions.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/PluginsVersions.java index 50792e42ca..abd95ed842 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/PluginsVersions.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/feign/dto/PluginsVersions.java @@ -28,6 +28,8 @@ public class PluginsVersions { private String age; private String gender; + private String pose; private String detector; private String calculator; + private String mask; } diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/service/FacesRestApiClient.java b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/service/FacesRestApiClient.java index 134706de16..2f589c277d 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/service/FacesRestApiClient.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/sdk/faces/service/FacesRestApiClient.java @@ -24,10 +24,9 @@ public class FacesRestApiClient implements FacesApiClient { private final FacesFeignClient feignClient; @Override - public FindFacesResponse findFaces(final MultipartFile photo, final Integer faceLimit, final Double thresholdC, - final String facePlugins) { + public FindFacesResponse findFaces(final MultipartFile photo, final Integer faceLimit, final Double thresholdC, final String facePlugins, final Boolean detectFaces) { try { - return feignClient.findFaces(photo, faceLimit, thresholdC, facePlugins); + return feignClient.findFaces(photo, faceLimit, thresholdC, facePlugins, detectFaces); } catch (FeignException.BadRequest ex) { throw new NoFacesFoundException(); } catch (FeignException e) { @@ -36,13 +35,14 @@ public FindFacesResponse findFaces(final MultipartFile photo, final Integer face } @Override - public FindFacesResponse findFacesBase64(String imageAsBase64, Integer faceLimit, Double thresholdC, String facePlugins) { + public FindFacesResponse findFacesBase64(final String imageAsBase64, final Integer faceLimit, final Double thresholdC, final String facePlugins, final Boolean detectFaces) { try { return feignClient.findFacesBase64( new FindFacesRequest(imageAsBase64), faceLimit, thresholdC, - facePlugins + facePlugins, + detectFaces ); } catch (FeignException.BadRequest ex) { throw new NoFacesFoundException(); @@ -52,17 +52,16 @@ public FindFacesResponse findFacesBase64(String imageAsBase64, Integer faceLimit } @Override - public FindFacesResponse findFacesWithCalculator(final MultipartFile photo, final Integer faceLimit, final Double thresholdC, - final String facePlugins) { - return findWithCalculator(photo, null, faceLimit, thresholdC, facePlugins); + public FindFacesResponse findFacesWithCalculator(final MultipartFile photo, final Integer faceLimit, final Double thresholdC, final String facePlugins, final Boolean detectFaces) { + return findWithCalculator(photo, null, faceLimit, thresholdC, facePlugins, detectFaces); } @Override - public FindFacesResponse findFacesBase64WithCalculator(String imageAsBase64, Integer faceLimit, Double thresholdC, String facePlugins) { - return findWithCalculator(null, imageAsBase64, faceLimit, thresholdC, facePlugins); + public FindFacesResponse findFacesBase64WithCalculator(final String imageAsBase64, final Integer faceLimit, final Double thresholdC, final String facePlugins, final Boolean detectFaces) { + return findWithCalculator(null, imageAsBase64, faceLimit, thresholdC, facePlugins, detectFaces); } - private FindFacesResponse findWithCalculator(final MultipartFile photo, final String imageAsBase64, Integer faceLimit, Double thresholdC, String facePlugins) { + private FindFacesResponse findWithCalculator(final MultipartFile photo, final String imageAsBase64, final Integer faceLimit, final Double thresholdC, final String facePlugins, final Boolean detectFaces) { try { String finalFacePlugins; if (StringUtils.isNotBlank(facePlugins)) { @@ -76,13 +75,14 @@ private FindFacesResponse findWithCalculator(final MultipartFile photo, final St } if (photo != null) { - return feignClient.findFaces(photo, faceLimit, thresholdC, finalFacePlugins); + return feignClient.findFaces(photo, faceLimit, thresholdC, finalFacePlugins, detectFaces); } else { return feignClient.findFacesBase64( new FindFacesRequest(imageAsBase64), faceLimit, thresholdC, - finalFacePlugins + finalFacePlugins, + detectFaces ); } } catch (FeignException.BadRequest ex) { diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/ApperyStatisticsClient.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/ApperyStatisticsClient.java index aa69226981..5f4bb93180 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/ApperyStatisticsClient.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/ApperyStatisticsClient.java @@ -17,5 +17,4 @@ public interface ApperyStatisticsClient { @PostMapping(path = "/statistics_faces") void create(@RequestHeader(value = DATABASE_ID_HEADER) String apiKey, StatisticsFacesEntity entity); - -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/StatisticsFacesEntity.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/StatisticsFacesEntity.java index 1c019c8d13..291cd68161 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/StatisticsFacesEntity.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/StatisticsFacesEntity.java @@ -18,4 +18,4 @@ public class StatisticsFacesEntity { @JsonProperty("faces_range") private String range; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/StatisticsGeneralEntity.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/StatisticsGeneralEntity.java index 4ebf445295..564070d464 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/StatisticsGeneralEntity.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/feign/StatisticsGeneralEntity.java @@ -17,4 +17,4 @@ public class StatisticsGeneralEntity { @JsonProperty("action_name") private StatisticsType actionName; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/global/Constants.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/global/Constants.java index a8e793a54a..bd079328a6 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/system/global/Constants.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/global/Constants.java @@ -26,4 +26,4 @@ public class Constants { public static final String DET_PROB_THRESHOLD = "det_prob_threshold"; public static final String FACE_PLUGINS = "face_plugins"; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/global/EnvironmentProperties.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/global/EnvironmentProperties.java index e184d90f3b..4100f6075a 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/system/global/EnvironmentProperties.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/global/EnvironmentProperties.java @@ -41,4 +41,4 @@ public static final class ServerInfo { public enum ServerType { PYTHON } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/global/ImageProperties.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/global/ImageProperties.java index 855fe9cefa..158ccc4256 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/system/global/ImageProperties.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/global/ImageProperties.java @@ -33,4 +33,4 @@ public class ImageProperties { private final List types; private boolean saveImagesToDB; -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/global/RegExConstants.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/global/RegExConstants.java new file mode 100644 index 0000000000..0116d970d4 --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/global/RegExConstants.java @@ -0,0 +1,10 @@ +package com.exadel.frs.commonservice.system.global; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class RegExConstants { + + public static final String ALLOWED_SPECIAL_CHARACTERS = "[^;/\\\\]+"; + public static final String PROHIBITED_SPECIAL_CHARACTERS = "[;/\\\\]+"; +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/FacesToSubjectsMigration.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/FacesToSubjectsMigration.java index 4a408c9f9e..32075574e4 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/FacesToSubjectsMigration.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/FacesToSubjectsMigration.java @@ -57,4 +57,4 @@ public void setFileOpener(ResourceAccessor resourceAccessor) { public ValidationErrors validate(Database database) { return null; } -} \ No newline at end of file +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/TransactionalFaceMigration.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/TransactionalFaceMigration.java index 00b0614aed..1b6f5d3971 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/TransactionalFaceMigration.java +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/TransactionalFaceMigration.java @@ -35,8 +35,6 @@ public void doFaceMigrationInTransaction(String apiKey, String faceId, String fa ); log.debug("Inserted subject with id {}", subjectId); - } else { - // subject for current face already exists } UUID imgId = null; diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/customchange/RemoveSpecialCharactersCustomChange.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/customchange/RemoveSpecialCharactersCustomChange.java new file mode 100644 index 0000000000..edc705007d --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/customchange/RemoveSpecialCharactersCustomChange.java @@ -0,0 +1,162 @@ +package com.exadel.frs.commonservice.system.liquibase.customchange; + +import static com.exadel.frs.commonservice.system.global.RegExConstants.PROHIBITED_SPECIAL_CHARACTERS; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import liquibase.change.custom.CustomTaskChange; +import liquibase.database.Database; +import liquibase.database.jvm.JdbcConnection; +import liquibase.exception.CustomChangeException; +import liquibase.exception.DatabaseException; +import liquibase.exception.SetupException; +import liquibase.exception.ValidationErrors; +import liquibase.repackaged.org.apache.commons.text.StringSubstitutor; +import liquibase.resource.ResourceAccessor; +import lombok.Getter; +import lombok.Setter; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +@Getter +@Setter +public class RemoveSpecialCharactersCustomChange implements CustomTaskChange { + + private static final Pattern SPECIAL_CHARACTERS_PATTERN = Pattern.compile(PROHIBITED_SPECIAL_CHARACTERS); + + private static final String COUNT_SQL_TEMPLATE = "SELECT COUNT(*) FROM ${table}"; + private static final String SELECT_SQL_TEMPLATE = "SELECT ${primaryKey}, ${target} FROM ${table}"; + private static final String UPDATE_SQL_TEMPLATE = "UPDATE ${table} SET ${target} = ? WHERE ${primaryKey} = ?"; + + private static final String PREFIX = "${"; + private static final String SUFFIX = "}"; + + private String table; + private String primaryKeyColumn; + private String targetColumn; + + @Override + public void execute(final Database database) throws CustomChangeException { + try { + JdbcConnection connection = (JdbcConnection) database.getConnection(); + List targets = getAllTargets(connection); + List cleanedTargets = cleanTargets(targets); + updateTargets(targets, cleanedTargets, connection); + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new CustomChangeException(e); + } + } + + private List getAllTargets(final JdbcConnection connection) throws DatabaseException, SQLException { + String sql = StringSubstitutor.replace(SELECT_SQL_TEMPLATE, getParams(), PREFIX, SUFFIX); + try (Statement statement = connection.createStatement()) { + int rowCount = getRowCount(connection); + List targets = new ArrayList<>(rowCount); + ResultSet resultSet = statement.executeQuery(sql); + + while (resultSet.next()) { + Object primaryKey = resultSet.getObject(1); + String value = resultSet.getString(2).trim(); + Target target = new Target(primaryKey, value); + targets.add(target); + } + + return targets; + } + } + + private int getRowCount(final JdbcConnection connection) throws DatabaseException, SQLException { + String sql = StringSubstitutor.replace(COUNT_SQL_TEMPLATE, getParams(), PREFIX, SUFFIX); + try (Statement statement = connection.createStatement()) { + ResultSet resultSet = statement.executeQuery(sql); + resultSet.next(); + return resultSet.getInt(1); + } + } + + private List cleanTargets(final List targets) { + List cleanedTargets = new ArrayList<>(); + for (Target target : targets) { + Matcher matcher = SPECIAL_CHARACTERS_PATTERN.matcher(target.getValue()); + if (matcher.find()) { + Target cleanedTarget = new Target( + target.getPrimaryKey(), + matcher.replaceAll("") + ); + cleanedTargets.add(cleanedTarget); + } + } + return cleanedTargets; + } + + private void updateTargets(final List targets, + final List cleanedTargets, + final JdbcConnection connection) throws DatabaseException, SQLException { + String sql = StringSubstitutor.replace(UPDATE_SQL_TEMPLATE, getParams(), PREFIX, SUFFIX); + List targetValues = targets.stream().map(Target::getValue).collect(Collectors.toList()); + + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (Target cleanedTarget : cleanedTargets) { + Object primaryKey = cleanedTarget.getPrimaryKey(); + String value = validateTargetValue(cleanedTarget.getValue(), targetValues) + ? cleanedTarget.getValue() + : UUID.randomUUID().toString(); + + statement.setString(1, value); + statement.setObject(2, primaryKey); + statement.addBatch(); + } + statement.executeBatch(); + } + } + + private boolean validateTargetValue(final String value, final List values) { + return StringUtils.isNotBlank(value) && !values.contains(value); + } + + private Map getParams() { + return Map.of( + "primaryKey", primaryKeyColumn, + "target", targetColumn, + "table", table + ); + } + + @Override + public String getConfirmationMessage() { + return null; + } + + @Override + public void setUp() throws SetupException { + + } + + @Override + public void setFileOpener(final ResourceAccessor resourceAccessor) { + + } + + @Override + public ValidationErrors validate(final Database database) { + return null; + } + + @Value + private static class Target { + + Object primaryKey; + String value; + } +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/customchange/SetOAuthTokenExpirationCustomChange.java b/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/customchange/SetOAuthTokenExpirationCustomChange.java new file mode 100644 index 0000000000..28f8432a8a --- /dev/null +++ b/java/common/src/main/java/com/exadel/frs/commonservice/system/liquibase/customchange/SetOAuthTokenExpirationCustomChange.java @@ -0,0 +1,106 @@ +package com.exadel.frs.commonservice.system.liquibase.customchange; + +import static java.time.ZoneOffset.UTC; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import liquibase.change.custom.CustomTaskChange; +import liquibase.database.Database; +import liquibase.database.jvm.JdbcConnection; +import liquibase.exception.CustomChangeException; +import liquibase.exception.DatabaseException; +import liquibase.exception.SetupException; +import liquibase.exception.ValidationErrors; +import liquibase.resource.ResourceAccessor; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@Setter +public class SetOAuthTokenExpirationCustomChange implements CustomTaskChange { + + private static final String REFRESH_TOKEN = "refresh_token"; + + private static final String SET_ACCESS_TOKEN_EXPIRATION_SQL = "UPDATE oauth_access_token SET expiration = ? WHERE client_id = ?"; + private static final String SET_REFRESH_TOKEN_EXPIRATION_SQL = "UPDATE oauth_refresh_token SET expiration = ? WHERE token_id IN (SELECT refresh_token FROM oauth_access_token WHERE client_id = ?)"; + + private String clientId; + private Integer accessTokenValidity; + private Integer refreshTokenValidity; + private String authorizedGrantTypes; + + @Override + public void execute(final Database database) throws CustomChangeException { + try { + JdbcConnection connection = (JdbcConnection) database.getConnection(); + setAccessTokenExpiration(connection); + if (authorizedGrantTypes.contains(REFRESH_TOKEN)) { + setRefreshTokenExpiration(connection); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new CustomChangeException(e); + } + } + + private void setAccessTokenExpiration(final JdbcConnection connection) + throws DatabaseException, SQLException { + int updateCount = setTokenExpiration( + connection, + SET_ACCESS_TOKEN_EXPIRATION_SQL, + accessTokenValidity + ); + log.info( + "Updated {} access tokens for client {}", + updateCount, + clientId + ); + } + + private void setRefreshTokenExpiration(final JdbcConnection connection) + throws DatabaseException, SQLException { + int updateCount = setTokenExpiration( + connection, + SET_REFRESH_TOKEN_EXPIRATION_SQL, + refreshTokenValidity + ); + log.info( + "Updated {} refresh tokens for client {}", + updateCount, + clientId + ); + } + + private int setTokenExpiration(final JdbcConnection connection, final String sql, final int tokenValidity) + throws DatabaseException, SQLException { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + Timestamp expiration = Timestamp.valueOf(LocalDateTime.now(UTC).plusSeconds(tokenValidity)); + statement.setTimestamp(1, expiration); + statement.setString(2, clientId); + return statement.executeUpdate(); + } + } + + @Override + public String getConfirmationMessage() { + return null; + } + + @Override + public void setUp() throws SetupException { + + } + + @Override + public void setFileOpener(final ResourceAccessor resourceAccessor) { + + } + + @Override + public ValidationErrors validate(final Database database) { + return null; + } +} diff --git a/java/pom.xml b/java/pom.xml index 8bee428f4b..c3d50cdbf6 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -8,8 +8,8 @@ org.springframework.boot spring-boot-starter-parent - 2.3.4.RELEASE - + 2.5.13 + com.exadel @@ -24,33 +24,51 @@ - 11 - 11 - 11 - 2.3.4.RELEASE + 17 + 17 + + 17 + + 2.5.13 1.1.1.RELEASE - 1.3.1.Final - 1.18.12 + 2.5.0.RELEASE + 2020.0.5 + + 1.5.2.Final + 1.18.20 2.9.2 - frs - 0.9.1 + 0.11.2 3.8.0 - 10.7.4 - 3.11 - 2.2.3.RELEASE + 11.8 + 3.12.0 2.7.2 1.1.1 - 5.3 - 2.9.13 + 7.2 + 2.16.2 1.0.0-beta7 - 11 - linux-x86_64 - 4.3.5 + 4.8.0 0.8.9 + 1.6.2 + 9.1.6 + 1.6.10 + + 3.8.4 + 2.22.2 + 3.8.1 + + linux-x86_64 + frs + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud.version} + pom + import + org.liquibase liquibase-core @@ -88,6 +106,11 @@ spring-security-oauth2-autoconfigure ${spring.boot.version} + + org.springframework.security.oauth + spring-security-oauth2 + ${spring-security-oauth2.version} + org.springframework.security spring-security-jwt @@ -95,14 +118,9 @@ io.jsonwebtoken - jjwt + jjwt-api ${io.jsonwebtoken.version} - - org.springframework.cloud - spring-cloud-starter-openfeign - ${spring-cloud-starter-openfeign.version} - io.github.openfeign feign-core @@ -159,6 +177,11 @@ ${nd4j.version} ${nd4j.classifier} + + com.impossibl.pgjdbc-ng + pgjdbc-ng + ${pgjdbc-ng.version} + org.springframework.boot spring-boot-starter-validation @@ -167,7 +190,13 @@ io.zonky.test embedded-database-spring-test - 1.6.2 + ${embedded-database-spring-test.version} + test + + + com.icegreen + greenmail-junit5 + ${greenmail-junit5.version} test @@ -179,12 +208,12 @@ org.liquibase liquibase-maven-plugin - 3.8.4 + ${liquibase-maven-plugin.version} org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + ${maven-compiler-plugin.version} @@ -210,6 +239,14 @@ + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + -Duser.timezone=UTC + + org.springframework.boot spring-boot-maven-plugin @@ -217,4 +254,5 @@ - \ No newline at end of file + + diff --git a/load-tests/README.md b/load-tests/README.md new file mode 100644 index 0000000000..714d077001 --- /dev/null +++ b/load-tests/README.md @@ -0,0 +1,46 @@ +# K6 load tests for CompreFace services +Each folder inside `tests` folder contains a separate loading test. +To run tests, first you need to build an image: +``` +cd ./docker +docker build -t k6tests . +``` + +Then you can run all tests or define a list of tests (as TESTS env variable): +``` +# run only face_verify and recognize tests +docker run \ + -e TESTS="face_verify;recognize" \ + -e HOSTNAME="http://myhost:8082" \ + -e INFLUXDB_HOSTNAME="http://myinfluxdbhost:8086" + -e DB_CONNECTION_STRING="user=postgres password=postgres port=5432 dbname=frs host=mydbhost sslmode=disable" \ + k6tests +``` +``` +# run all tests +docker run \ + -e HOSTNAME="http://myhost:8082" \ + -e INFLUXDB_HOSTNAME="http://myinfluxdbhost:8086" + -e DB_CONNECTION_STRING="user=postgres password=postgres port=5432 dbname=frs host=mydbhost sslmode=disable" \ + k6tests +``` + +Any test from `tests` folder follows those steps: +1. Apply db_init.sql to database +2. Run recognition test according to `scenarios` defined in the script +3. Apply db_truncate.sql to database + + +### Run command details +``` +docker run + --env IMAGES="./faces/FACE_512KB.jpg;./faces/FACE_1024KB.jpg" + --env HOSTNAME="" + --env INFLUXDB_HOSTNAME="" + --env DB_CONNECTION_STRING="user=postgres password= port=5432 dbname=frs host= sslmode=disable" + +``` +`IMAGES` list of images fot test (if images are needed for the test) +`HOSTNAME` hostname of test server +`INFLUXDB_HOSTNAME` hostname of influxdb +`DB_CONNECTION_STRING` DB connection string, template is *"user=mydbuser password=mydbpass port=5432 dbname=mydbname host=mydbhost sslmode=disable"* diff --git a/load-tests/docker/Dockerfile b/load-tests/docker/Dockerfile new file mode 100644 index 0000000000..9ce815cbe4 --- /dev/null +++ b/load-tests/docker/Dockerfile @@ -0,0 +1,19 @@ +FROM alpine:3.13.4 + +ENV K6_HOME="/data/k6" \ +# run all tests by default + TESTS="" \ + VUS="1" \ + ITERATIONS="1" \ + DURATION="2m" \ + HOSTNAME="http://localhost:8000" \ + INFLUXDB_HOSTNAME="http://localhost:8086" \ + DB_CONNECTION_STRING="user=postgres password=postgres port=5432 dbname=frs host=localhost sslmode=disable" + +RUN mkdir -p ${K6_HOME} +WORKDIR ${K6_HOME} +ADD . . +RUN chmod +x ./entrypoint.sh \ + && chmod +x ./k6 + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/load-tests/docker/entrypoint.sh b/load-tests/docker/entrypoint.sh new file mode 100644 index 0000000000..0cafa17552 --- /dev/null +++ b/load-tests/docker/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +FOLDERS="" + +if [ -z "$TESTS" ] +then + echo "TESTS is empty, run all tests" + FOLDERS=$(ls -F tests | tr "/ " "\n") +else + echo "TESTS to run: $TESTS" + FOLDERS=$(echo $TESTS | tr ";" "\n") +fi + +for test_folder in $FOLDERS +do + echo "************************************************************************" + echo "********************************** $test_folder" + echo "************************************************************************" + + chmod +x ./tests/${test_folder}/loadtest.k6.js + ./k6 run \ + --insecure-skip-tls-verify \ + --vus ${VUS} \ + --iterations ${ITERATIONS} \ + --duration ${DURATION} \ + -e HOSTNAME="$HOSTNAME" \ + -e DB_CONNECTION_STRING="$DB_CONNECTION_STRING" \ + --out influxdb=${INFLUXDB_HOSTNAME}/${test_folder} \ + ./tests/${test_folder}/loadtest.k6.js +done diff --git a/load-tests/docker/k6 b/load-tests/docker/k6 new file mode 100644 index 0000000000..0c9271640e Binary files /dev/null and b/load-tests/docker/k6 differ diff --git a/load-tests/docker/tests/detect/db_init.sql b/load-tests/docker/tests/detect/db_init.sql new file mode 100644 index 0000000000..97518e05e4 --- /dev/null +++ b/load-tests/docker/tests/detect/db_init.sql @@ -0,0 +1,2 @@ +INSERT INTO app(id, name, guid, api_key) VALUES (9223372036854775801, 'TEST_K6_1', 'a1111a11-ae6d-4636-8b47-49754ca2f54b', '6f348698-6985-4296-914e-a5e3270cfad1'); +INSERT INTO model(id, name, guid, api_key, app_id, type) VALUES (9223372036854775801, 'D_SERVICE', 'm1111a11-ae6d-4636-8b47-49754ca2f54b', '1f348698-6985-4296-914e-a5e3270cfad7', 9223372036854775801, 'D'); diff --git a/load-tests/docker/tests/detect/db_truncate.sql b/load-tests/docker/tests/detect/db_truncate.sql new file mode 100644 index 0000000000..24d371aac6 --- /dev/null +++ b/load-tests/docker/tests/detect/db_truncate.sql @@ -0,0 +1,2 @@ +DELETE FROM model WHERE id IN ('9223372036854775801'); +DELETE FROM app WHERE id IN ('9223372036854775801'); diff --git a/load-tests/docker/tests/detect/faces/FACE_1.jpg b/load-tests/docker/tests/detect/faces/FACE_1.jpg new file mode 100644 index 0000000000..5be702561d Binary files /dev/null and b/load-tests/docker/tests/detect/faces/FACE_1.jpg differ diff --git a/load-tests/docker/tests/detect/formdata.js b/load-tests/docker/tests/detect/formdata.js new file mode 100644 index 0000000000..e11c16c172 --- /dev/null +++ b/load-tests/docker/tests/detect/formdata.js @@ -0,0 +1,93 @@ +/* + * FormData polyfill for k6 + * Copyright (C) 2021 Load Impact + * License: MIT + * + * This simplifies the creation of multipart/form-data requests from k6 scripts. + * It was adapted from the original version by Rob Wu[1] to remove references of + * XMLHttpRequest and File related code which isn't supported in k6. + * + * [1]: https://gist.github.com/Rob--W/8b5adedd84c0d36aba64 + **/ + +if (exports.FormData) { + // Don't replace FormData if it already exists + return; +} + +// Export variable to the global scope +exports.FormData = FormData; + +function FormData() { + // Force a Constructor + if (!(this instanceof FormData)) return new FormData(); + // Generate a random boundary - This must be unique with respect to the + // form's contents. + this.boundary = '------RWWorkerFormDataBoundary' + Math.random().toString(36); + this.parts = []; + + /** + * Internal method. Convert input to a byte array. + * @param inp String | ArrayBuffer | Uint8Array Input + */ + this.__toByteArray = function(inp) { + var arr = []; + var i = 0, len; + if (typeof inp === 'string') { + for (len = inp.length; i < len; ++i) + arr.push(inp.charCodeAt(i) & 0xff); + } else if (inp && inp.byteLength) {/*If ArrayBuffer or typed array */ + if (!('byteOffset' in inp)) /* If ArrayBuffer, wrap in view */ + inp = new Uint8Array(inp); + for (len = inp.byteLength; i < len; ++i) + arr.push(inp[i] & 0xff); + } + return arr; + }; +} + +/** + * @param fieldName String Form field name + * @param data object|string An object or string field value. + * + * If data is an object, it should match the structure of k6's http.FileData + * object (returned by http.file()) and consist of: + * @param data.data String|Array|ArrayBuffer File data + * @param data.filename String Optional file name + * @param data.content_type String Optional content type, default is application/octet-stream + **/ +FormData.prototype.append = function(fieldName, data) { + if (arguments.length < 2) { + throw new SyntaxError('Not enough arguments'); + } + var file = data; + if (typeof data === 'string') { + file = {data: data, content_type: 'text/plain'}; + } + this.parts.push({field: fieldName, file: file}); +}; + +/** + * Return the assembled request body as an ArrayBuffer. + **/ +FormData.prototype.body = function() { + var body = []; + var barr = this.__toByteArray('--' + this.boundary + '\r\n'); + for (var i=0; i < this.parts.length; i++) { + Array.prototype.push.apply(body, barr); + var p = this.parts[i]; + var cd = 'Content-Disposition: form-data; name="' + p.field + '"'; + if (p.file.filename) { + cd += '; filename="' + p.file.filename.replace(/"/g,'%22') + '"'; + } + cd += '\r\nContent-Type: ' + + (p.file.content_type || 'application/octet-stream') + + '\r\n\r\n'; + Array.prototype.push.apply(body, this.__toByteArray(cd)); + var data = Array.isArray(p.file.data) ? p.file.data : this.__toByteArray(p.file.data); + Array.prototype.push.apply(body, data); + Array.prototype.push.apply(body, this.__toByteArray('\r\n')); + } + Array.prototype.push.apply(body, this.__toByteArray('--' + this.boundary + '--\r\n')); + return new Uint8Array(body).buffer; +}; diff --git a/load-tests/docker/tests/detect/loadtest.k6.js b/load-tests/docker/tests/detect/loadtest.k6.js new file mode 100644 index 0000000000..8369a7fd26 --- /dev/null +++ b/load-tests/docker/tests/detect/loadtest.k6.js @@ -0,0 +1,84 @@ +import sql from "k6/x/sql" +import http from "k6/http" +import { check } from 'k6' +import { FormData } from './formdata.js' + + +const PREDEFINED_IMAGES = [ + "./faces/FACE_1.jpg" +] + +const XAPIKEY = '1f348698-6985-4296-914e-a5e3270cfad7' +const IMAGE_PATHS = __ENV.IMAGES ? __ENV.IMAGES.split(';') : PREDEFINED_IMAGES +const IMAGES = getImageFiles(IMAGE_PATHS) +const REQUEST_TIMEOUT = 360000 +const db_init_sql = open("./db_init.sql") +const db_truncate_sql = open("./db_truncate.sql") +const db = sql.open("postgres", __ENV.DB_CONNECTION_STRING) + + +export let options = { + scenarios: { + my_awesome_api_test: { + executor: 'constant-vus', + vus: 8, + duration: '1m', // possible opts "Xs" (X seconds), "Xm" (X minutes), "Xh" (X hours), "Xd" (X days) + }, + }, + thresholds: { + http_req_duration: ['p(99)<3000'], // 99% of requests must complete below 3s + }, +}; + +export function setup() { + console.log("DB: " + __ENV.DB_CONNECTION_STRING) + console.log("Host: " + __ENV.HOSTNAME) + + execute_sql(db_init_sql) + return {} +} + +export function teardown(data) { + execute_sql(db_truncate_sql) + db.close() +} + +export default function(data) { + let response = verify(IMAGES[IMAGE_PATHS[0]], IMAGES[IMAGE_PATHS[1]]) + check(response, { + 'status 200': (r) => r.status === 200, + 'probability': (r) => r.body.indexOf('probability') !== -1, + }) +} + +function verify(image_file) { + let url = __ENV.HOSTNAME + '/api/v1/detection/detect' + + const fd = new FormData() + fd.append('file', http.file(image_file, 'file.jpg', 'image/jpeg')) + fd.append('limit', '0') + fd.append('prediction_count', '1') + + let headers = { + 'Content-Type': 'multipart/form-data; boundary=' + fd.boundary, + 'x-api-key': XAPIKEY, + } + + let params = {headers: headers, timeout: REQUEST_TIMEOUT} + + return http.post(url, fd.body(), params) +} + +function getImageFiles(image_paths) { + let image_files = {} + + for (let index = 0; index < image_paths.length; ++index) { + image_files[IMAGE_PATHS[index]] = open(image_paths[index], 'b') + } + + return image_files +} + +export function execute_sql(sql_string) { + db.exec(sql_string) +} diff --git a/load-tests/docker/tests/face_verify/db_init.sql b/load-tests/docker/tests/face_verify/db_init.sql new file mode 100644 index 0000000000..a6aee9e684 --- /dev/null +++ b/load-tests/docker/tests/face_verify/db_init.sql @@ -0,0 +1,4 @@ +INSERT INTO app(id, name, guid, api_key) VALUES (9223372036854775802, 'TEST_K6_1', 'a1111a11-ae6d-4636-8b47-49754ca2f54b', '6f348698-6985-4296-914e-a5e3270cfad1'); +INSERT INTO model(id, name, guid, api_key, app_id, type) VALUES (9223372036854775802, 'R_SERVICE', 'm1111a11-ae6d-4636-8b47-49754ca2f54b', '1f348698-6985-4296-914e-a5e3270cfad7', 9223372036854775802, 'R'); +INSERT INTO subject(id, api_key, subject_name) VALUES ('00000000-0000-0000-0000-000000000100', '1f348698-6985-4296-914e-a5e3270cfad7', 'Test Face'); +INSERT INTO embedding(id, subject_id, calculator, embedding) VALUES ('00000000-0000-0000-0000-000000000101', '00000000-0000-0000-0000-000000000100', 'Facenet2018', '{-0.03097902610898018,-0.0615910142660141,0.004267971031367779,-0.029088227078318596,0.01998152956366539,0.004408995155245066,7.079810020513833E-4,0.02055450901389122,0.0559232234954834,-0.017034798860549927,0.042523302137851715,0.020132875069975853,0.030375173315405846,0.011112676933407784,0.0037292146589607,0.138917937874794,0.02335701324045658,-0.007966035045683384,-0.004250305239111185,0.009982270188629627,0.018235566094517708,0.026537297293543816,0.009864152409136295,0.03696277365088463,-0.0013352083042263985,-0.044565897434949875,-0.08291416615247726,0.040734779089689255,0.024890407919883728,0.022149255499243736,-0.029522130265831947,-0.07688562572002411,0.0022794187534600496,-0.030212679877877235,0.005983452778309584,0.03557029739022255,0.08241281658411026,-0.011958680115640163,0.04503229260444641,-0.013083958066999912,0.04853236302733421,0.004872147925198078,0.06146423891186714,0.00959888193756342,7.77583394665271E-4,0.029161933809518814,0.028342798352241516,0.017955709248781204,6.902238237671554E-4,0.0901801660656929,0.02867206744849682,0.013741467148065567,-0.025805111974477768,-0.034532591700553894,0.11199014633893967,0.01385575719177723,-0.0067737591452896595,-0.07397060096263885,0.02677309513092041,-8.832994499243796E-4,0.025572745129466057,-0.013665265403687954,-0.03542732447385788,0.030406856909394264,-0.04648316279053688,-0.025098171085119247,0.0029712743125855923,0.04422789812088013,0.05144299194216728,-0.02187458425760269,-0.05024687200784683,-0.03312273323535919,0.004544287454336882,-0.014914292842149734,0.033138781785964966,0.0024307849816977978,0.07039214670658112,0.02909677103161812,-0.020016904920339584,-0.043430157005786896,-0.02863958477973938,-0.03646234795451164,-0.04638441279530525,-0.031245585530996323,-0.0011027348227798939,-0.01152162067592144,0.0022944423835724592,-0.01422145776450634,-0.03055015578866005,-0.05160767585039139,-0.0016859190072864294,-0.03544624522328377,0.01885906048119068,0.07672908902168274,-0.13038083910942078,0.00959916040301323,0.015406432561576366,-0.03114278055727482,0.08482049405574799,0.02668837644159794,-0.07692013680934906,0.014024175703525543,9.628521511331201E-4,-0.03849990293383598,-0.017628947272896767,-0.05767654627561569,-0.06516523659229279,-0.04151972755789757,0.06223363056778908,0.05276373401284218,-0.019681034609675407,0.017606817185878754,0.04694725573062897,0.03001430444419384,0.07560744881629944,0.04251708835363388,0.04462055116891861,-0.039653319865465164,-0.01805521920323372,-0.048499152064323425,0.07164400815963745,0.027550937607884407,0.02784995548427105,-0.02225816622376442,0.08171040564775467,0.056947603821754456,-0.013058015145361423,-0.011959037743508816,0.018647925928235054,0.06659363955259323,0.0312957689166069,-0.004100460559129715,-0.05170748755335808,0.02758433297276497,-0.022638481110334396,-0.01709084026515484,-0.03948419541120529,0.011781134642660618,0.11459562182426453,-0.07447241991758347,0.0024151368997991085,-0.055515725165605545,0.08348339051008224,-0.02752760984003544,0.036186665296554565,-0.010092778131365776,0.05034494772553444,0.01330098882317543,-0.0350995697081089,-0.046944476664066315,-0.01404506154358387,-0.08276499062776566,0.05728853493928909,-0.03734053671360016,-0.037630025297403336,-0.07849573343992233,-0.03490936756134033,0.016933001577854156,-0.004286612384021282,0.015985945239663124,-0.06359440088272095,-0.0691550225019455,-0.010922783985733986,-0.025971949100494385,0.05667164549231529,0.024773385375738144,0.011372092179954052,-0.004382803104817867,-0.01723073236644268,-0.05401792749762535,0.03298680856823921,-0.010194086469709873,0.0670757070183754,0.06018674746155739,0.09611929953098297,-0.025354627519845963,0.04338163137435913,0.0886528342962265,-0.004735982045531273,0.059263937175273895,-0.016539841890335083,-0.013872893527150154,0.006561543792486191,-0.10404457896947861,-0.03412356600165367,-0.06073751300573349,-0.007314895745366812,-0.043480437248945236,-0.026068342849612236,-0.039584334939718246,-0.014128518290817738,0.03757422789931297,-9.311420581070706E-5,1.2821046766475774E-5,-0.030188199132680893,0.024010242894291878,0.034948792308568954,0.05881265923380852,-0.03413897007703781,-0.04142145439982414,-4.8736209282651544E-4,0.014667666517198086,0.12242266535758972,-0.1146383285522461,-0.04143412038683891,-0.014596131630241871,0.032614875584840775,0.04335426911711693,-2.8282226412557065E-4,-0.006311641540378332,0.005719346925616264,0.046196430921554565,0.042517758905887604,-0.020725378766655922,0.021425427868962288,0.0289135854691267,0.004072137176990509,-0.10577140003442764,-0.05501950532197952,-0.08584046363830566,0.006458538584411144,-0.01338035985827446,0.06478573381900787,-0.011796667240560055,-0.04147907346487045,-0.02094469778239727,-0.031569357961416245,-0.03236137330532074,0.08992817252874374,-0.04124844819307327,-0.033292073756456375,-0.05728865787386894,0.012902868911623955,0.06219278275966644,0.020194895565509796,0.02651306800544262,0.0023552083875983953,-0.002733596134930849,-2.1032063523307443E-4,0.07116362452507019,-0.017394550144672394,0.01675480417907238,0.06930588185787201,-0.014390580356121063,-0.027466265484690666,0.024026738479733467,-8.810216677375138E-4,0.009994743391871452,-0.04741652309894562,-0.0015808623284101486,0.0019500961061567068,0.013211345300078392,-0.030923625454306602,-0.02319086156785488,0.018938366323709488,-0.054130274802446365,-0.07673376053571701,-0.08749832957983017,-0.010204199701547623,-0.05128772184252739,-0.07677685469388962,-0.0657929927110672,-0.013882198370993137,-0.013863911852240562,-0.026858016848564148,-0.01601695641875267,-0.029884064570069313,-0.007367642130702734,-0.044770050793886185,-0.027539538219571114,-0.007074476219713688,0.0644354298710823,-0.01714247465133667,-0.017399225383996964,-0.02700674906373024,-0.037100814282894135,0.009443148970603943,0.014645954594016075,0.019855879247188568,0.021477364003658295,-0.029836110770702362,-0.041720785200595856,0.03683692216873169,-0.0013433679705485702,0.10861194133758545,-0.003858777927234769,0.003286811290308833,0.030585426837205887,-0.005279154051095247,0.013851318508386612,0.00990385189652443,0.07188807427883148,-0.05747256055474281,0.09912173449993134,0.03855753690004349,0.1248202845454216,-0.06831198185682297,-0.04168170318007469,-0.004079767037183046,-0.027480948716402054,0.0038354636635631323,0.07853391021490097,0.02883046120405197,0.05123080685734749,0.0072994716465473175,0.025991329923272133,0.03288726136088371,0.0049530453979969025,-0.007984044030308723,-0.0332929864525795,0.03470882028341293,0.002079891739413142,4.806758661288768E-4,0.0019827657379209995,0.018021220341324806,0.027885528281331062,0.04088839516043663,0.003150214208289981,-0.009342361241579056,-0.03738715872168541,-0.03189055249094963,-0.007134676910936832,0.010295611806213856,0.006902702618390322,-0.045737821608781815,0.08199654519557953,-0.07050422579050064,-0.003301135962828994,-0.043714094907045364,-0.017302319407463074,-0.016130201518535614,-0.06463446468114853,0.004058522172272205,-0.03614160791039467,-0.002204370917752385,0.0074641346000134945,0.05470031499862671,0.06308937817811966,-0.02573964186012745,0.06271446496248245,0.03812674432992935,-0.030722420662641525,-0.0618540458381176,0.09032673388719559,0.05936750769615173,-0.05599411949515343,-0.0011097081005573273,-0.006562475115060806,-0.010946985334157944,-0.03339753672480583,0.0674029216170311,-0.05376448109745979,0.01989232562482357,0.049486685544252396,0.03792495280504227,0.01375060435384512,-0.019614605233073235,-0.07926559448242188,-0.05760283023118973,0.004906968213617802,0.022030964493751526,0.009283466264605522,0.015307007357478142,0.022669976577162743,-0.02801590785384178,0.053507015109062195,-0.07906698435544968,0.0030029546469449997,-9.463618043810129E-4,-0.002631227020174265,0.012956338003277779,0.012722086161375046,-0.054254092276096344,-0.05653367564082146,-0.04602457210421562,0.02016562595963478,-0.03821982815861702,-0.07559265196323395,0.0839204341173172,0.013991083949804306,-0.020466169342398643,0.08207625895738602,0.07137484848499298,0.07780403643846512,0.058997318148612976,-0.005137298721820116,0.0032198624685406685,0.006167932879179716,-0.06888788938522339,-0.08890273422002792,0.07478892803192139,-0.03843782842159271,0.026146968826651573,0.013510963879525661,-0.04340091720223427,-0.03885417804121971,0.07456088066101074,-0.019653474912047386,0.046087365597486496,-0.010241392068564892,0.051461536437273026,0.002797784050926566,0.04813159629702568,-0.01651453971862793,-0.01912376657128334,-0.046075187623500824,0.010030128993093967,-0.0065520829521119595,0.02425885945558548,0.029688838869333267,0.032781701534986496,0.0038261853624135256,-0.007221803069114685,-0.053272493183612823,0.013579913415014744,-0.004602286498993635,0.02403074875473976,-0.0021762654650956392,0.12324189394712448,-0.0030621869955211878,0.0024159536696970463,-0.055524345487356186,-0.01536979153752327,-0.07954566925764084,-0.008501201868057251,0.016443949192762375,0.007139910012483597,0.03859306126832962,-0.059025175869464874,0.011182818561792374,0.02492438815534115,0.0721653625369072,-0.016363391652703285,0.03127870336174965,0.04162347689270973,0.0220218263566494,0.11782315373420715,-0.0045939055271446705,-0.07514511793851852,0.05786045268177986,0.03115895576775074,-0.027585044503211975,-0.01934988610446453,-0.04066358134150505,0.006281341891735792,-0.05053640902042389,0.008617371320724487,-0.03563675656914711,0.07130882889032364,0.049607764929533005,-0.009763461537659168,-0.04726259782910347,-0.04994232952594757,0.07731116563081741,0.014653232879936695,-0.05857133865356445,0.005273112561553717,-0.12458549439907074,-0.11482326686382294,0.015383974649012089,-0.030921299010515213,-0.06462979316711426,-0.009868007153272629,0.06621633470058441,-0.04428550973534584,-0.07898282259702682,0.0642974004149437,-0.036258917301893234,-0.0784270316362381,0.0280571561306715,-0.005694199353456497,0.04966180399060249,-0.029891980811953545,0.018052807077765465,-0.023499151691794395,4.963010433129966E-4,-0.043104514479637146,0.015557142905890942,-0.013176372274756432,-0.013165372423827648,-0.0916002169251442,-0.028345486149191856,-0.022008372470736504,-0.01910957135260105,-0.056190989911556244,0.04007869213819504,0.03817107528448105,0.0040502166375517845,-0.05597272887825966,0.008966329507529736,0.036289799958467484,-0.007604269776493311,0.027250315994024277,0.02756989747285843,0.019304433837532997,0.06925279647111893,0.034056905657052994,0.01882236823439598,-0.07679253071546555,3.256367635913193E-4,0.04564471170306206,0.029979072511196136,-0.022164393216371536,-0.07272813469171524,0.04419924318790436,-0.006878197658807039,7.24894751328975E-4,0.00554495258256793,0.050300490111112595,0.06040569022297859,-0.030833374708890915,-0.05043018236756325}'); diff --git a/load-tests/docker/tests/face_verify/db_truncate.sql b/load-tests/docker/tests/face_verify/db_truncate.sql new file mode 100644 index 0000000000..22feca46a3 --- /dev/null +++ b/load-tests/docker/tests/face_verify/db_truncate.sql @@ -0,0 +1,4 @@ +DELETE FROM embedding WHERE id IN ('00000000-0000-0000-0000-000000000101'); +DELETE FROM subject WHERE id IN ('00000000-0000-0000-0000-000000000100'); +DELETE FROM model WHERE id IN ('9223372036854775802'); +DELETE FROM app WHERE id IN ('9223372036854775802'); diff --git a/load-tests/docker/tests/face_verify/faces/FACE_1.jpg b/load-tests/docker/tests/face_verify/faces/FACE_1.jpg new file mode 100644 index 0000000000..5be702561d Binary files /dev/null and b/load-tests/docker/tests/face_verify/faces/FACE_1.jpg differ diff --git a/load-tests/docker/tests/face_verify/formdata.js b/load-tests/docker/tests/face_verify/formdata.js new file mode 100644 index 0000000000..e11c16c172 --- /dev/null +++ b/load-tests/docker/tests/face_verify/formdata.js @@ -0,0 +1,93 @@ +/* + * FormData polyfill for k6 + * Copyright (C) 2021 Load Impact + * License: MIT + * + * This simplifies the creation of multipart/form-data requests from k6 scripts. + * It was adapted from the original version by Rob Wu[1] to remove references of + * XMLHttpRequest and File related code which isn't supported in k6. + * + * [1]: https://gist.github.com/Rob--W/8b5adedd84c0d36aba64 + **/ + +if (exports.FormData) { + // Don't replace FormData if it already exists + return; +} + +// Export variable to the global scope +exports.FormData = FormData; + +function FormData() { + // Force a Constructor + if (!(this instanceof FormData)) return new FormData(); + // Generate a random boundary - This must be unique with respect to the + // form's contents. + this.boundary = '------RWWorkerFormDataBoundary' + Math.random().toString(36); + this.parts = []; + + /** + * Internal method. Convert input to a byte array. + * @param inp String | ArrayBuffer | Uint8Array Input + */ + this.__toByteArray = function(inp) { + var arr = []; + var i = 0, len; + if (typeof inp === 'string') { + for (len = inp.length; i < len; ++i) + arr.push(inp.charCodeAt(i) & 0xff); + } else if (inp && inp.byteLength) {/*If ArrayBuffer or typed array */ + if (!('byteOffset' in inp)) /* If ArrayBuffer, wrap in view */ + inp = new Uint8Array(inp); + for (len = inp.byteLength; i < len; ++i) + arr.push(inp[i] & 0xff); + } + return arr; + }; +} + +/** + * @param fieldName String Form field name + * @param data object|string An object or string field value. + * + * If data is an object, it should match the structure of k6's http.FileData + * object (returned by http.file()) and consist of: + * @param data.data String|Array|ArrayBuffer File data + * @param data.filename String Optional file name + * @param data.content_type String Optional content type, default is application/octet-stream + **/ +FormData.prototype.append = function(fieldName, data) { + if (arguments.length < 2) { + throw new SyntaxError('Not enough arguments'); + } + var file = data; + if (typeof data === 'string') { + file = {data: data, content_type: 'text/plain'}; + } + this.parts.push({field: fieldName, file: file}); +}; + +/** + * Return the assembled request body as an ArrayBuffer. + **/ +FormData.prototype.body = function() { + var body = []; + var barr = this.__toByteArray('--' + this.boundary + '\r\n'); + for (var i=0; i < this.parts.length; i++) { + Array.prototype.push.apply(body, barr); + var p = this.parts[i]; + var cd = 'Content-Disposition: form-data; name="' + p.field + '"'; + if (p.file.filename) { + cd += '; filename="' + p.file.filename.replace(/"/g,'%22') + '"'; + } + cd += '\r\nContent-Type: ' + + (p.file.content_type || 'application/octet-stream') + + '\r\n\r\n'; + Array.prototype.push.apply(body, this.__toByteArray(cd)); + var data = Array.isArray(p.file.data) ? p.file.data : this.__toByteArray(p.file.data); + Array.prototype.push.apply(body, data); + Array.prototype.push.apply(body, this.__toByteArray('\r\n')); + } + Array.prototype.push.apply(body, this.__toByteArray('--' + this.boundary + '--\r\n')); + return new Uint8Array(body).buffer; +}; diff --git a/load-tests/docker/tests/face_verify/loadtest.k6.js b/load-tests/docker/tests/face_verify/loadtest.k6.js new file mode 100644 index 0000000000..854309fdb5 --- /dev/null +++ b/load-tests/docker/tests/face_verify/loadtest.k6.js @@ -0,0 +1,87 @@ +import sql from "k6/x/sql"; +import http from "k6/http"; +import { check } from 'k6'; +import { FormData } from './formdata.js'; + +const PREDEFINED_IMAGES = [ + "./faces/FACE_1.jpg" +] + +const XAPIKEY = '1f348698-6985-4296-914e-a5e3270cfad7' +const images = getImageFiles() +const REQUEST_TIMEOUT = 360000 +const db_init_sql = open("./db_init.sql") +const db_truncate_sql = open("./db_truncate.sql") +const db = sql.open("postgres", __ENV.DB_CONNECTION_STRING) + + +export let options = { + scenarios: { + my_awesome_api_test: { + executor: 'constant-vus', + vus: 8, + duration: '1m', // possible opts "Xs" (X seconds), "Xm" (X minutes), "Xh" (X hours), "Xd" (X days) + }, + }, + thresholds: { + http_req_duration: ['p(99)<3000'], // 99% of requests must complete below 3s + }, +}; + +export function setup() { + console.log("DB: " + __ENV.DB_CONNECTION_STRING) + console.log("Host: " + __ENV.HOSTNAME) + + execute_sql(db_init_sql) + return {} +} + +export function teardown(data) { + execute_sql(db_truncate_sql) + db.close() +} + +export default function(data) { + let image_paths = Object.keys(images) + let image_path = image_paths.length < __VU ? image_paths[__VU % image_paths.length] : image_paths[__VU - 1] + console.log(image_path) + let response = recognize(images[image_path]) + check(response, { + 'status 200': (r) => r.status === 200, + 'similarity': (r) => r.body.indexOf('similarity') !== -1, + }) + +} + +function recognize(image_file) { + let url = __ENV.HOSTNAME + '/api/v1/recognition/faces/00000000-0000-0000-0000-000000000101/verify' + + const fd = new FormData(); + fd.append('file', http.file(image_file, 'file.jpg', 'image/jpeg')); + + let headers = { + 'Content-Type': 'multipart/form-data; boundary=' + fd.boundary, + 'x-api-key': XAPIKEY, + } + + let params = {headers: headers, timeout: REQUEST_TIMEOUT} + + return http.post(url, fd.body(), params) +} + +function getImageFiles() { + let image_files = {} + let image_paths = __ENV.IMAGES + ? __ENV.IMAGES.split(';') + : PREDEFINED_IMAGES + + for (let index = 0; index < image_paths.length; ++index) { + image_files[image_paths[index]] = open(image_paths[index], 'b') + } + + return image_files +} + +export function execute_sql(sql_string) { + db.exec(sql_string) +} diff --git a/load-tests/docker/tests/recognize/db_init.sql b/load-tests/docker/tests/recognize/db_init.sql new file mode 100644 index 0000000000..3b9456704e --- /dev/null +++ b/load-tests/docker/tests/recognize/db_init.sql @@ -0,0 +1,2 @@ +INSERT INTO app(id, name, guid, api_key) VALUES (9223372036854775803, 'TEST_K6_1', 'a1111a11-ae6d-4636-8b47-49754ca2f54b', '6f348698-6985-4296-914e-a5e3270cfad1'); +INSERT INTO model(id, name, guid, api_key, app_id, type) VALUES (9223372036854775803, 'R_SERVICE', 'm1111a11-ae6d-4636-8b47-49754ca2f54b', '1f348698-6985-4296-914e-a5e3270cfad7', 9223372036854775803, 'R'); diff --git a/load-tests/docker/tests/recognize/db_truncate.sql b/load-tests/docker/tests/recognize/db_truncate.sql new file mode 100644 index 0000000000..eb664ce75c --- /dev/null +++ b/load-tests/docker/tests/recognize/db_truncate.sql @@ -0,0 +1,2 @@ +DELETE FROM model WHERE id IN ('9223372036854775803'); +DELETE FROM app WHERE id IN ('9223372036854775803'); diff --git a/load-tests/docker/tests/recognize/faces/FACE_1.jpg b/load-tests/docker/tests/recognize/faces/FACE_1.jpg new file mode 100644 index 0000000000..5be702561d Binary files /dev/null and b/load-tests/docker/tests/recognize/faces/FACE_1.jpg differ diff --git a/load-tests/docker/tests/recognize/formdata.js b/load-tests/docker/tests/recognize/formdata.js new file mode 100644 index 0000000000..e11c16c172 --- /dev/null +++ b/load-tests/docker/tests/recognize/formdata.js @@ -0,0 +1,93 @@ +/* + * FormData polyfill for k6 + * Copyright (C) 2021 Load Impact + * License: MIT + * + * This simplifies the creation of multipart/form-data requests from k6 scripts. + * It was adapted from the original version by Rob Wu[1] to remove references of + * XMLHttpRequest and File related code which isn't supported in k6. + * + * [1]: https://gist.github.com/Rob--W/8b5adedd84c0d36aba64 + **/ + +if (exports.FormData) { + // Don't replace FormData if it already exists + return; +} + +// Export variable to the global scope +exports.FormData = FormData; + +function FormData() { + // Force a Constructor + if (!(this instanceof FormData)) return new FormData(); + // Generate a random boundary - This must be unique with respect to the + // form's contents. + this.boundary = '------RWWorkerFormDataBoundary' + Math.random().toString(36); + this.parts = []; + + /** + * Internal method. Convert input to a byte array. + * @param inp String | ArrayBuffer | Uint8Array Input + */ + this.__toByteArray = function(inp) { + var arr = []; + var i = 0, len; + if (typeof inp === 'string') { + for (len = inp.length; i < len; ++i) + arr.push(inp.charCodeAt(i) & 0xff); + } else if (inp && inp.byteLength) {/*If ArrayBuffer or typed array */ + if (!('byteOffset' in inp)) /* If ArrayBuffer, wrap in view */ + inp = new Uint8Array(inp); + for (len = inp.byteLength; i < len; ++i) + arr.push(inp[i] & 0xff); + } + return arr; + }; +} + +/** + * @param fieldName String Form field name + * @param data object|string An object or string field value. + * + * If data is an object, it should match the structure of k6's http.FileData + * object (returned by http.file()) and consist of: + * @param data.data String|Array|ArrayBuffer File data + * @param data.filename String Optional file name + * @param data.content_type String Optional content type, default is application/octet-stream + **/ +FormData.prototype.append = function(fieldName, data) { + if (arguments.length < 2) { + throw new SyntaxError('Not enough arguments'); + } + var file = data; + if (typeof data === 'string') { + file = {data: data, content_type: 'text/plain'}; + } + this.parts.push({field: fieldName, file: file}); +}; + +/** + * Return the assembled request body as an ArrayBuffer. + **/ +FormData.prototype.body = function() { + var body = []; + var barr = this.__toByteArray('--' + this.boundary + '\r\n'); + for (var i=0; i < this.parts.length; i++) { + Array.prototype.push.apply(body, barr); + var p = this.parts[i]; + var cd = 'Content-Disposition: form-data; name="' + p.field + '"'; + if (p.file.filename) { + cd += '; filename="' + p.file.filename.replace(/"/g,'%22') + '"'; + } + cd += '\r\nContent-Type: ' + + (p.file.content_type || 'application/octet-stream') + + '\r\n\r\n'; + Array.prototype.push.apply(body, this.__toByteArray(cd)); + var data = Array.isArray(p.file.data) ? p.file.data : this.__toByteArray(p.file.data); + Array.prototype.push.apply(body, data); + Array.prototype.push.apply(body, this.__toByteArray('\r\n')); + } + Array.prototype.push.apply(body, this.__toByteArray('--' + this.boundary + '--\r\n')); + return new Uint8Array(body).buffer; +}; diff --git a/load-tests/docker/tests/recognize/loadtest.k6.js b/load-tests/docker/tests/recognize/loadtest.k6.js new file mode 100644 index 0000000000..bad4225514 --- /dev/null +++ b/load-tests/docker/tests/recognize/loadtest.k6.js @@ -0,0 +1,83 @@ +import sql from "k6/x/sql"; +import http from "k6/http"; +import { check } from 'k6'; +import { FormData } from './formdata.js'; + +const PREDEFINED_IMAGES = [ + "./faces/FACE_1.jpg" +] + +const XAPIKEY = '1f348698-6985-4296-914e-a5e3270cfad7' +const IMAGE_PATHS = __ENV.IMAGES ? __ENV.IMAGES.split(';') : PREDEFINED_IMAGES +const IMAGES = getImageFiles(IMAGE_PATHS) +const REQUEST_TIMEOUT = 360000 +const db_init_sql = open("./db_init.sql") +const db_truncate_sql = open("./db_truncate.sql") +const db = sql.open("postgres", __ENV.DB_CONNECTION_STRING) + + +export let options = { + scenarios: { + my_awesome_api_test: { + executor: 'constant-vus', + vus: 8, + duration: '1m', // possible opts "Xs" (X seconds), "Xm" (X minutes), "Xh" (X hours), "Xd" (X days) + }, + }, + thresholds: { + http_req_duration: ['p(99)<3000'], // 99% of requests must complete below 3s + }, +}; + +export function setup() { + console.log("DB: " + __ENV.DB_CONNECTION_STRING) + console.log("Host: " + __ENV.HOSTNAME) + + execute_sql(db_init_sql) + return {} +} + +export function teardown(data) { + execute_sql(db_truncate_sql) + db.close() +} + +export default function(data) { + let response = verify(IMAGES[IMAGE_PATHS[0]], IMAGES[IMAGE_PATHS[1]]) + check(response, { + 'status 200': (r) => r.status === 200, + 'probability': (r) => r.body.indexOf('probability') !== -1, + }) +} + +function verify(image_file) { + let url = __ENV.HOSTNAME + '/api/v1/recognition/recognize' + + const fd = new FormData(); + fd.append('file', http.file(image_file, 'file.jpg', 'image/jpeg')); + fd.append('limit', '0'); + fd.append('prediction_count', '1'); + + let headers = { + 'Content-Type': 'multipart/form-data; boundary=' + fd.boundary, + 'x-api-key': XAPIKEY, + } + + let params = {headers: headers, timeout: REQUEST_TIMEOUT} + + return http.post(url, fd.body(), params) +} + +function getImageFiles(image_paths) { + let image_files = {} + + for (let index = 0; index < image_paths.length; ++index) { + image_files[IMAGE_PATHS[index]] = open(image_paths[index], 'b') + } + + return image_files +} + +export function execute_sql(sql_string) { + db.exec(sql_string) +} diff --git a/load-tests/docker/tests/verify/db_init.sql b/load-tests/docker/tests/verify/db_init.sql new file mode 100644 index 0000000000..92c77d9175 --- /dev/null +++ b/load-tests/docker/tests/verify/db_init.sql @@ -0,0 +1,2 @@ +INSERT INTO app(id, name, guid, api_key) VALUES (9223372036854775804, 'TEST_K6_1', 'a1111a11-ae6d-4636-8b47-49754ca2f54b', '6f348698-6985-4296-914e-a5e3270cfad1'); +INSERT INTO model(id, name, guid, api_key, app_id, type) VALUES (9223372036854775804, 'V_SERVICE', 'm1111a11-ae6d-4636-8b47-49754ca2f54b', '1f348698-6985-4296-914e-a5e3270cfad7', 9223372036854775804, 'V'); diff --git a/load-tests/docker/tests/verify/db_truncate.sql b/load-tests/docker/tests/verify/db_truncate.sql new file mode 100644 index 0000000000..513b51ac3b --- /dev/null +++ b/load-tests/docker/tests/verify/db_truncate.sql @@ -0,0 +1,2 @@ +DELETE FROM model WHERE id IN ('9223372036854775804'); +DELETE FROM app WHERE id IN ('9223372036854775804'); diff --git a/load-tests/docker/tests/verify/faces/FACE_1.jpg b/load-tests/docker/tests/verify/faces/FACE_1.jpg new file mode 100644 index 0000000000..1f089cf5be Binary files /dev/null and b/load-tests/docker/tests/verify/faces/FACE_1.jpg differ diff --git a/load-tests/docker/tests/verify/faces/FACE_2.jpg b/load-tests/docker/tests/verify/faces/FACE_2.jpg new file mode 100644 index 0000000000..ff50a2f469 Binary files /dev/null and b/load-tests/docker/tests/verify/faces/FACE_2.jpg differ diff --git a/load-tests/docker/tests/verify/faces/FACE_3.jpg b/load-tests/docker/tests/verify/faces/FACE_3.jpg new file mode 100644 index 0000000000..e9b9a8f628 Binary files /dev/null and b/load-tests/docker/tests/verify/faces/FACE_3.jpg differ diff --git a/load-tests/docker/tests/verify/formdata.js b/load-tests/docker/tests/verify/formdata.js new file mode 100644 index 0000000000..e11c16c172 --- /dev/null +++ b/load-tests/docker/tests/verify/formdata.js @@ -0,0 +1,93 @@ +/* + * FormData polyfill for k6 + * Copyright (C) 2021 Load Impact + * License: MIT + * + * This simplifies the creation of multipart/form-data requests from k6 scripts. + * It was adapted from the original version by Rob Wu[1] to remove references of + * XMLHttpRequest and File related code which isn't supported in k6. + * + * [1]: https://gist.github.com/Rob--W/8b5adedd84c0d36aba64 + **/ + +if (exports.FormData) { + // Don't replace FormData if it already exists + return; +} + +// Export variable to the global scope +exports.FormData = FormData; + +function FormData() { + // Force a Constructor + if (!(this instanceof FormData)) return new FormData(); + // Generate a random boundary - This must be unique with respect to the + // form's contents. + this.boundary = '------RWWorkerFormDataBoundary' + Math.random().toString(36); + this.parts = []; + + /** + * Internal method. Convert input to a byte array. + * @param inp String | ArrayBuffer | Uint8Array Input + */ + this.__toByteArray = function(inp) { + var arr = []; + var i = 0, len; + if (typeof inp === 'string') { + for (len = inp.length; i < len; ++i) + arr.push(inp.charCodeAt(i) & 0xff); + } else if (inp && inp.byteLength) {/*If ArrayBuffer or typed array */ + if (!('byteOffset' in inp)) /* If ArrayBuffer, wrap in view */ + inp = new Uint8Array(inp); + for (len = inp.byteLength; i < len; ++i) + arr.push(inp[i] & 0xff); + } + return arr; + }; +} + +/** + * @param fieldName String Form field name + * @param data object|string An object or string field value. + * + * If data is an object, it should match the structure of k6's http.FileData + * object (returned by http.file()) and consist of: + * @param data.data String|Array|ArrayBuffer File data + * @param data.filename String Optional file name + * @param data.content_type String Optional content type, default is application/octet-stream + **/ +FormData.prototype.append = function(fieldName, data) { + if (arguments.length < 2) { + throw new SyntaxError('Not enough arguments'); + } + var file = data; + if (typeof data === 'string') { + file = {data: data, content_type: 'text/plain'}; + } + this.parts.push({field: fieldName, file: file}); +}; + +/** + * Return the assembled request body as an ArrayBuffer. + **/ +FormData.prototype.body = function() { + var body = []; + var barr = this.__toByteArray('--' + this.boundary + '\r\n'); + for (var i=0; i < this.parts.length; i++) { + Array.prototype.push.apply(body, barr); + var p = this.parts[i]; + var cd = 'Content-Disposition: form-data; name="' + p.field + '"'; + if (p.file.filename) { + cd += '; filename="' + p.file.filename.replace(/"/g,'%22') + '"'; + } + cd += '\r\nContent-Type: ' + + (p.file.content_type || 'application/octet-stream') + + '\r\n\r\n'; + Array.prototype.push.apply(body, this.__toByteArray(cd)); + var data = Array.isArray(p.file.data) ? p.file.data : this.__toByteArray(p.file.data); + Array.prototype.push.apply(body, data); + Array.prototype.push.apply(body, this.__toByteArray('\r\n')); + } + Array.prototype.push.apply(body, this.__toByteArray('--' + this.boundary + '--\r\n')); + return new Uint8Array(body).buffer; +}; diff --git a/load-tests/docker/tests/verify/loadtest.k6.js b/load-tests/docker/tests/verify/loadtest.k6.js new file mode 100644 index 0000000000..39ea90ea01 --- /dev/null +++ b/load-tests/docker/tests/verify/loadtest.k6.js @@ -0,0 +1,89 @@ +import sql from "k6/x/sql"; +import http from "k6/http"; +import { check } from 'k6'; +import { FormData } from './formdata.js'; + +const PREDEFINED_IMAGES = [ + "./faces/FACE_1.jpg", + "./faces/FACE_2.jpg", +] + +const XAPIKEY = '1f348698-6985-4296-914e-a5e3270cfad7' +const images = getImageFiles() +const REQUEST_TIMEOUT = 360000 +const db_init_sql = open("./db_init.sql") +const db_truncate_sql = open("./db_truncate.sql") +const db = sql.open("postgres", __ENV.DB_CONNECTION_STRING) + + +export let options = { + scenarios: { + my_awesome_api_test: { + executor: 'constant-vus', + vus: 8, + duration: '1m', // possible opts "Xs" (X seconds), "Xm" (X minutes), "Xh" (X hours), "Xd" (X days) + }, + }, + thresholds: { + http_req_duration: ['p(99)<3000'], // 99% of requests must complete below 3s + }, +}; + +export function setup() { + console.log("DB: " + __ENV.DB_CONNECTION_STRING) + console.log("Host: " + __ENV.HOSTNAME) + + execute_sql(db_init_sql) + return {} +} + +export function teardown(data) { + execute_sql(db_truncate_sql) + db.close() +} + +export default function(data) { + let image_paths = Object.keys(images) + let image_path = image_paths.length < __VU ? image_paths[__VU % image_paths.length] : image_paths[__VU - 1] + console.log(image_path) + let response = recognize(images[image_path]) + check(response, { + 'status 200': (r) => r.status === 200, + 'similarity': (r) => r.body.indexOf('similarity') !== -1, + }) + +} + +function recognize(image_file) { + let url = __ENV.HOSTNAME + '/api/v1/verification/verify' + + const fd = new FormData(); + fd.append('source_image', http.file(image_file, 'source_image.jpg', 'image/jpeg')); + fd.append('target_image', http.file(image_file, 'target_image.jpg', 'image/jpeg')); + + let headers = { + 'Content-Type': 'multipart/form-data; boundary=' + fd.boundary, + 'x-api-key': XAPIKEY, + } + + let params = {headers: headers, timeout: REQUEST_TIMEOUT} + + return http.post(url, fd.body(), params) +} + +function getImageFiles() { + let image_files = {} + let image_paths = __ENV.IMAGES + ? __ENV.IMAGES.split(';') + : PREDEFINED_IMAGES + + for (let index = 0; index < image_paths.length; ++index) { + image_files[image_paths[index]] = open(image_paths[index], 'b') + } + + return image_files +} + +export function execute_sql(sql_string) { + db.exec(sql_string) +} diff --git a/load-tests/grafana/docker-compose.yml b/load-tests/grafana/docker-compose.yml new file mode 100644 index 0000000000..0ae5181e55 --- /dev/null +++ b/load-tests/grafana/docker-compose.yml @@ -0,0 +1,39 @@ +version: "3.5" + +services: + influxdb: + image: influxdb:1.8.10-alpine + container_name: influxdb + restart: always + ports: + - "8086:8086" + networks: + - grafana + volumes: + - influxdb-data:/var/lib/influxdb + + grafana: + image: grafana/grafana:9.3.0 + container_name: grafana + restart: always + depends_on: + - influxdb + ports: + - "3000:3000" + networks: + - grafana + volumes: + - grafana-data:/var/lib/grafana + - ./provisioning:/etc/grafana/provisioning + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_BASIC_ENABLED=false + - GF_DISABLE_LOGIN_FORM=true + +networks: + grafana: + +volumes: + influxdb-data: + grafana-data: diff --git a/load-tests/grafana/provisioning/dashboards/dashboard.yml b/load-tests/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000000..9ff1b48391 --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,35 @@ +apiVersion: 1 + +providers: + - name: 'recognize_influxdb_datasource' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards + - name: 'detect_influxdb_datasource' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards + - name: 'verify_influxdb_datasource' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards + - name: 'face_verify_influxdb_datasource' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/load-tests/grafana/provisioning/dashboards/detect-k6-load-testing-results-by-groups.json b/load-tests/grafana/provisioning/dashboards/detect-k6-load-testing-results-by-groups.json new file mode 100644 index 0000000000..c991730649 --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/detect-k6-load-testing-results-by-groups.json @@ -0,0 +1,2956 @@ +{ + "__inputs": [ + { + "name": "detect_influxdb_datasource", + "label": "Detect k6 Load Testing Results", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.3.6" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "table-old", + "name": "Table (old)", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool by groups, using the InfluxDB exporter.Based on https://grafana.com/dashboards/10660", + "editable": true, + "gnetId": 13719, + "graphTooltip": 2, + "id": null, + "iteration": 1611037943041, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1157", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 2 + }, + "hiddenSeries": false, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_reqs", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1009", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1010", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:745", + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"errors\" WHERE $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:752", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:753", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 2 + }, + "hiddenSeries": false, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1522", + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1529", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1530", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "decimals": null, + "description": "Grouped by 1 sec intervals and group name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "height": "250px", + "hiddenSeries": false, + "id": 71, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "group" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time per group)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2444", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2445", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "decimals": null, + "description": "Grouped by 1 sec intervals and group name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "height": "250px", + "hiddenSeries": false, + "id": 74, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "name" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time per tag)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2444", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2445", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "description": "Grouped by 1 sec intervals", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "height": "250px", + "hiddenSeries": false, + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1318", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1319", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "detect_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "heatmap": {}, + "height": "250px", + "hideZeroBuckets": false, + "highlightCards": true, + "id": 8, + "interval": ">1s", + "legend": { + "show": false + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 29 + }, + "height": "50px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (count)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 29 + }, + "height": "50px", + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (min)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 29 + }, + "height": "50px", + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (med)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 29 + }, + "height": "50px", + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (max)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 29 + }, + "height": "50px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (mean)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 29 + }, + "height": "50px", + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (p95)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [], + "datasource": "detect_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 19, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 67, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "P95", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "percentile", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkUrl": "", + "pattern": "url", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "min", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "max", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "mean", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "median", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "min", + "dsType": "influxdb", + "expr": "", + "format": "table", + "groupBy": [ + { + "params": [ + "group" + ], + "type": "tag" + } + ], + "hide": false, + "intervalFactor": 2, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "URLs and latencies (By Group)", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": "detect_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 19, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 73, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "P95", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "percentile", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkUrl": "", + "pattern": "url", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "min", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "max", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "mean", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "median", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "min", + "dsType": "influxdb", + "expr": "", + "format": "table", + "groupBy": [ + { + "params": [ + "name" + ], + "type": "tag" + } + ], + "hide": false, + "intervalFactor": 2, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "URLs and latencies (By Name)", + "transform": "table", + "type": "table-old" + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "k6", + "load testing" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "http_req_duration", + "value": [ + "http_req_duration" + ] + }, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "http_req_duration", + "value": "http_req_duration" + }, + { + "selected": false, + "text": "http_req_blocked", + "value": "http_req_blocked" + }, + { + "selected": false, + "text": "http_req_connecting", + "value": "http_req_connecting" + }, + { + "selected": false, + "text": "http_req_looking_up", + "value": "http_req_looking_up" + }, + { + "selected": false, + "text": "http_req_receiving", + "value": "http_req_receiving" + }, + { + "selected": false, + "text": "http_req_sending", + "value": "http_req_sending" + }, + { + "selected": false, + "text": "http_req_waiting", + "value": "http_req_waiting" + } + ], + "query": "http_req_duration,http_req_blocked,http_req_connecting,http_req_looking_up,http_req_receiving,http_req_sending,http_req_waiting", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": "*", + "current": {}, + "datasource": "detect_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "URL", + "multi": false, + "name": "URL", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"name\"", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "detect_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Group", + "multi": true, + "name": "Group", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"group\" WHERE $timeFilter", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "detect_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Tag", + "multi": true, + "name": "Tag", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"name\" WHERE $timeFilter", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "5m", + "30m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Detect k6 Load Testing Results By Groups", + "version": 38 +} \ No newline at end of file diff --git a/load-tests/grafana/provisioning/dashboards/detect-k6-load-testing-results.json b/load-tests/grafana/provisioning/dashboards/detect-k6-load-testing-results.json new file mode 100644 index 0000000000..612bc8c200 --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/detect-k6-load-testing-results.json @@ -0,0 +1,1617 @@ +{ + "__inputs": [ + { + "name": "detect_influxdb_datasource", + "label": "Detect k6 Load Testing Results", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool, using the InfluxDB exporter", + "editable": true, + "gnetId": 2587, + "graphTooltip": 2, + "hideControls": false, + "id": null, + "links": [], + "refresh": "5s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "fill": 1, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "fill": 1, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_reqs", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "fill": 1, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "fill": 1, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (mean)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (max)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT median(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (med)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (min)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (p90)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "detect_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (p95)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "detect_influxdb_datasource", + "description": "Grouped by 1 sec intervals", + "fill": 1, + "height": "250px", + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "detect_influxdb_datasource", + "heatmap": {}, + "height": "250px", + "highlightCards": true, + "id": 8, + "interval": ">1s", + "links": [], + "span": 6, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_req_duration", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"value\" FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketNumber": null, + "yBucketSize": null + } + ], + "repeat": "Measurement", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "$Measurement", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "http_req_duration + http_req_blocked", + "value": [ + "http_req_duration", + "http_req_blocked" + ] + }, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "http_req_duration", + "value": "http_req_duration" + }, + { + "selected": true, + "text": "http_req_blocked", + "value": "http_req_blocked" + }, + { + "selected": false, + "text": "http_req_connecting", + "value": "http_req_connecting" + }, + { + "selected": false, + "text": "http_req_looking_up", + "value": "http_req_looking_up" + }, + { + "selected": false, + "text": "http_req_receiving", + "value": "http_req_receiving" + }, + { + "selected": false, + "text": "http_req_sending", + "value": "http_req_sending" + }, + { + "selected": false, + "text": "http_req_waiting", + "value": "http_req_waiting" + } + ], + "query": "http_req_duration,http_req_blocked,http_req_connecting,http_req_looking_up,http_req_receiving,http_req_sending,http_req_waiting", + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Detect k6 Load Testing Results", + "version": 22 +} \ No newline at end of file diff --git a/load-tests/grafana/provisioning/dashboards/face-verify-k6-load-testing-results-by-groups.json b/load-tests/grafana/provisioning/dashboards/face-verify-k6-load-testing-results-by-groups.json new file mode 100644 index 0000000000..1998844179 --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/face-verify-k6-load-testing-results-by-groups.json @@ -0,0 +1,3017 @@ +{ + "__inputs": [ + { + "name": "face_verify_influxdb_datasource", + "label": "Face Verify k6 Load Testing Results", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.3.6" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "table-old", + "name": "Table (old)", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool by groups, using the InfluxDB exporter.Based on https://grafana.com/dashboards/10660", + "editable": true, + "gnetId": 13719, + "graphTooltip": 2, + "id": null, + "iteration": 1611037943041, + "links": [], + "panels": [ + { + "collapsed": true, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 68, + "panels": [ + { + "content": "### To choose which measurements to show, use the **Measurements** drop-down in the top left corner!\n\nK6 collects metrics for several phases during the HTTP request lifetime.\n\nThe combined delays for the application and network (send, wait, recieve) are available in this metric:\n * **http_req_duration** _( Total time for request, excluding time spent blocked (http_req_blocked), DNS lookup (http_req_looking_up) and TCP connect (http_req_connecting) time. )_\n\nThis metric can also be broken down into these metrics:\n * **http_req_tls_handshaking** _( Time spent handshaking TLS session with remote host. )_\n * **http_req_sending** _( Time spent sending data to remote host. )_\n * **http_req_waiting** _( Time spent waiting for response from remote host (a.k.a. \"time to first byte\", or \"TTFB\"). )_\n * **http_req_receiving** _( Time spent receiving response data from remote host. )_\n\nThese metrics measure delays and durations on the client:\n\n * **http_req_blocked** _( Time spent blocked (waiting for a free TCP connection slot) before initiating request. )_\n * **http_req_looking_up** _( Time spent looking up remote host name in DNS. )_\n * **http_req_connecting** _( Time spent establishing TCP connection to remote host. )_\n\n\n### View official K6.io documentation here: [https://docs.k6.io/docs/result-metrics][1]\n\n\n[1]: https://docs.k6.io/docs/result-metrics\n\n", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 66, + "links": [], + "mode": "markdown", + "title": "", + "transparent": true, + "type": "text" + } + ], + "repeat": null, + "title": "Documentation", + "type": "row" + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 69, + "panels": [], + "repeat": null, + "title": "K6 Test overview (all URLs)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1157", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 2 + }, + "hiddenSeries": false, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_reqs", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1009", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1010", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:745", + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"errors\" WHERE $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:752", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:753", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 2 + }, + "hiddenSeries": false, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1522", + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1529", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1530", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 70, + "panels": [], + "repeat": "Measurement", + "title": "$Measurement", + "type": "row" + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "decimals": null, + "description": "Grouped by 1 sec intervals and group name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "height": "250px", + "hiddenSeries": false, + "id": 71, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "group" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time per group)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2444", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2445", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "decimals": null, + "description": "Grouped by 1 sec intervals and group name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "height": "250px", + "hiddenSeries": false, + "id": 74, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "name" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time per tag)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2444", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2445", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "description": "Grouped by 1 sec intervals", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "height": "250px", + "hiddenSeries": false, + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1318", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1319", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "face_verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "heatmap": {}, + "height": "250px", + "hideZeroBuckets": false, + "highlightCards": true, + "id": 8, + "interval": ">1s", + "legend": { + "show": false + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 29 + }, + "height": "50px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (count)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 29 + }, + "height": "50px", + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (min)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 29 + }, + "height": "50px", + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (med)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 29 + }, + "height": "50px", + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (max)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 29 + }, + "height": "50px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (mean)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 29 + }, + "height": "50px", + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (p95)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [], + "datasource": "face_verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 19, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 67, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "P95", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "percentile", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkUrl": "", + "pattern": "url", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "min", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "max", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "mean", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "median", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "min", + "dsType": "influxdb", + "expr": "", + "format": "table", + "groupBy": [ + { + "params": [ + "group" + ], + "type": "tag" + } + ], + "hide": false, + "intervalFactor": 2, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "URLs and latencies (By Group)", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": "face_verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 19, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 73, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "P95", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "percentile", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkUrl": "", + "pattern": "url", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "min", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "max", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "mean", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "median", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "min", + "dsType": "influxdb", + "expr": "", + "format": "table", + "groupBy": [ + { + "params": [ + "name" + ], + "type": "tag" + } + ], + "hide": false, + "intervalFactor": 2, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "URLs and latencies (By Name)", + "transform": "table", + "type": "table-old" + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "k6", + "load testing" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "http_req_duration", + "value": [ + "http_req_duration" + ] + }, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "http_req_duration", + "value": "http_req_duration" + }, + { + "selected": false, + "text": "http_req_blocked", + "value": "http_req_blocked" + }, + { + "selected": false, + "text": "http_req_connecting", + "value": "http_req_connecting" + }, + { + "selected": false, + "text": "http_req_looking_up", + "value": "http_req_looking_up" + }, + { + "selected": false, + "text": "http_req_receiving", + "value": "http_req_receiving" + }, + { + "selected": false, + "text": "http_req_sending", + "value": "http_req_sending" + }, + { + "selected": false, + "text": "http_req_waiting", + "value": "http_req_waiting" + } + ], + "query": "http_req_duration,http_req_blocked,http_req_connecting,http_req_looking_up,http_req_receiving,http_req_sending,http_req_waiting", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": "*", + "current": {}, + "datasource": "face_verify_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "URL", + "multi": false, + "name": "URL", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"name\"", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "face_verify_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Group", + "multi": true, + "name": "Group", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"group\" WHERE $timeFilter", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "face_verify_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Tag", + "multi": true, + "name": "Tag", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"name\" WHERE $timeFilter", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "5m", + "30m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Face Verify k6 Load Testing Results By Groups", + "version": 38 +} \ No newline at end of file diff --git a/load-tests/grafana/provisioning/dashboards/face-verify-k6-load-testing-results.json b/load-tests/grafana/provisioning/dashboards/face-verify-k6-load-testing-results.json new file mode 100644 index 0000000000..f2e8e92028 --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/face-verify-k6-load-testing-results.json @@ -0,0 +1,1617 @@ +{ + "__inputs": [ + { + "name": "face_verify_influxdb_datasource", + "label": "Face Verify k6 Load Testing Results", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool, using the InfluxDB exporter", + "editable": true, + "gnetId": 2587, + "graphTooltip": 2, + "hideControls": false, + "id": null, + "links": [], + "refresh": "5s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "fill": 1, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "fill": 1, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_reqs", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "fill": 1, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "fill": 1, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (mean)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (max)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT median(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (med)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (min)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (p90)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "face_verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (p95)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "face_verify_influxdb_datasource", + "description": "Grouped by 1 sec intervals", + "fill": 1, + "height": "250px", + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "face_verify_influxdb_datasource", + "heatmap": {}, + "height": "250px", + "highlightCards": true, + "id": 8, + "interval": ">1s", + "links": [], + "span": 6, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_req_duration", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"value\" FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketNumber": null, + "yBucketSize": null + } + ], + "repeat": "Measurement", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "$Measurement", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "http_req_duration + http_req_blocked", + "value": [ + "http_req_duration", + "http_req_blocked" + ] + }, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "http_req_duration", + "value": "http_req_duration" + }, + { + "selected": true, + "text": "http_req_blocked", + "value": "http_req_blocked" + }, + { + "selected": false, + "text": "http_req_connecting", + "value": "http_req_connecting" + }, + { + "selected": false, + "text": "http_req_looking_up", + "value": "http_req_looking_up" + }, + { + "selected": false, + "text": "http_req_receiving", + "value": "http_req_receiving" + }, + { + "selected": false, + "text": "http_req_sending", + "value": "http_req_sending" + }, + { + "selected": false, + "text": "http_req_waiting", + "value": "http_req_waiting" + } + ], + "query": "http_req_duration,http_req_blocked,http_req_connecting,http_req_looking_up,http_req_receiving,http_req_sending,http_req_waiting", + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Face Verify k6 Load Testing Results", + "version": 22 +} \ No newline at end of file diff --git a/load-tests/grafana/provisioning/dashboards/recognize-k6-load-testing-results-by-groups.json b/load-tests/grafana/provisioning/dashboards/recognize-k6-load-testing-results-by-groups.json new file mode 100644 index 0000000000..978bf67876 --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/recognize-k6-load-testing-results-by-groups.json @@ -0,0 +1,3017 @@ +{ + "__inputs": [ + { + "name": "recognize_influxdb_datasource", + "label": "Recognize k6 Load Testing Results", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.3.6" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "table-old", + "name": "Table (old)", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool by groups, using the InfluxDB exporter.Based on https://grafana.com/dashboards/10660", + "editable": true, + "gnetId": 13719, + "graphTooltip": 2, + "id": null, + "iteration": 1611037943041, + "links": [], + "panels": [ + { + "collapsed": true, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 68, + "panels": [ + { + "content": "### To choose which measurements to show, use the **Measurements** drop-down in the top left corner!\n\nK6 collects metrics for several phases during the HTTP request lifetime.\n\nThe combined delays for the application and network (send, wait, recieve) are available in this metric:\n * **http_req_duration** _( Total time for request, excluding time spent blocked (http_req_blocked), DNS lookup (http_req_looking_up) and TCP connect (http_req_connecting) time. )_\n\nThis metric can also be broken down into these metrics:\n * **http_req_tls_handshaking** _( Time spent handshaking TLS session with remote host. )_\n * **http_req_sending** _( Time spent sending data to remote host. )_\n * **http_req_waiting** _( Time spent waiting for response from remote host (a.k.a. \"time to first byte\", or \"TTFB\"). )_\n * **http_req_receiving** _( Time spent receiving response data from remote host. )_\n\nThese metrics measure delays and durations on the client:\n\n * **http_req_blocked** _( Time spent blocked (waiting for a free TCP connection slot) before initiating request. )_\n * **http_req_looking_up** _( Time spent looking up remote host name in DNS. )_\n * **http_req_connecting** _( Time spent establishing TCP connection to remote host. )_\n\n\n### View official K6.io documentation here: [https://docs.k6.io/docs/result-metrics][1]\n\n\n[1]: https://docs.k6.io/docs/result-metrics\n\n", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 66, + "links": [], + "mode": "markdown", + "title": "", + "transparent": true, + "type": "text" + } + ], + "repeat": null, + "title": "Documentation", + "type": "row" + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 69, + "panels": [], + "repeat": null, + "title": "K6 Test overview (all URLs)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1157", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 2 + }, + "hiddenSeries": false, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_reqs", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1009", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1010", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:745", + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"errors\" WHERE $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:752", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:753", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 2 + }, + "hiddenSeries": false, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1522", + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1529", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1530", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 70, + "panels": [], + "repeat": "Measurement", + "title": "$Measurement", + "type": "row" + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "decimals": null, + "description": "Grouped by 1 sec intervals and group name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "height": "250px", + "hiddenSeries": false, + "id": 71, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "group" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time per group)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2444", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2445", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "decimals": null, + "description": "Grouped by 1 sec intervals and group name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "height": "250px", + "hiddenSeries": false, + "id": 74, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "name" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time per tag)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2444", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2445", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "description": "Grouped by 1 sec intervals", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "height": "250px", + "hiddenSeries": false, + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1318", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1319", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "recognize_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "heatmap": {}, + "height": "250px", + "hideZeroBuckets": false, + "highlightCards": true, + "id": 8, + "interval": ">1s", + "legend": { + "show": false + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 29 + }, + "height": "50px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (count)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 29 + }, + "height": "50px", + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (min)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 29 + }, + "height": "50px", + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (med)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 29 + }, + "height": "50px", + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (max)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 29 + }, + "height": "50px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (mean)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 29 + }, + "height": "50px", + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (p95)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [], + "datasource": "recognize_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 19, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 67, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "P95", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "percentile", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkUrl": "", + "pattern": "url", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "min", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "max", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "mean", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "median", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "min", + "dsType": "influxdb", + "expr": "", + "format": "table", + "groupBy": [ + { + "params": [ + "group" + ], + "type": "tag" + } + ], + "hide": false, + "intervalFactor": 2, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "URLs and latencies (By Group)", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": "recognize_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 19, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 73, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "P95", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "percentile", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkUrl": "", + "pattern": "url", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "min", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "max", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "mean", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "median", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "min", + "dsType": "influxdb", + "expr": "", + "format": "table", + "groupBy": [ + { + "params": [ + "name" + ], + "type": "tag" + } + ], + "hide": false, + "intervalFactor": 2, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "URLs and latencies (By Name)", + "transform": "table", + "type": "table-old" + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "k6", + "load testing" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "http_req_duration", + "value": [ + "http_req_duration" + ] + }, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "http_req_duration", + "value": "http_req_duration" + }, + { + "selected": false, + "text": "http_req_blocked", + "value": "http_req_blocked" + }, + { + "selected": false, + "text": "http_req_connecting", + "value": "http_req_connecting" + }, + { + "selected": false, + "text": "http_req_looking_up", + "value": "http_req_looking_up" + }, + { + "selected": false, + "text": "http_req_receiving", + "value": "http_req_receiving" + }, + { + "selected": false, + "text": "http_req_sending", + "value": "http_req_sending" + }, + { + "selected": false, + "text": "http_req_waiting", + "value": "http_req_waiting" + } + ], + "query": "http_req_duration,http_req_blocked,http_req_connecting,http_req_looking_up,http_req_receiving,http_req_sending,http_req_waiting", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": "*", + "current": {}, + "datasource": "recognize_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "URL", + "multi": false, + "name": "URL", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"name\"", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "recognize_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Group", + "multi": true, + "name": "Group", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"group\" WHERE $timeFilter", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "recognize_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Tag", + "multi": true, + "name": "Tag", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"name\" WHERE $timeFilter", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "5m", + "30m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Recognize k6 Load Testing Results By Groups", + "version": 38 +} \ No newline at end of file diff --git a/load-tests/grafana/provisioning/dashboards/recognize-k6-load-testing-results.json b/load-tests/grafana/provisioning/dashboards/recognize-k6-load-testing-results.json new file mode 100644 index 0000000000..561db96d8d --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/recognize-k6-load-testing-results.json @@ -0,0 +1,1617 @@ +{ + "__inputs": [ + { + "name": "recognize_influxdb_datasource", + "label": "Recognize k6 Load Testing Results", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool, using the InfluxDB exporter", + "editable": true, + "gnetId": 2587, + "graphTooltip": 2, + "hideControls": false, + "id": null, + "links": [], + "refresh": "5s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "fill": 1, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "fill": 1, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_reqs", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "fill": 1, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "fill": 1, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (mean)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (max)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT median(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (med)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (min)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (p90)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "recognize_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (p95)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "recognize_influxdb_datasource", + "description": "Grouped by 1 sec intervals", + "fill": 1, + "height": "250px", + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "recognize_influxdb_datasource", + "heatmap": {}, + "height": "250px", + "highlightCards": true, + "id": 8, + "interval": ">1s", + "links": [], + "span": 6, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_req_duration", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"value\" FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketNumber": null, + "yBucketSize": null + } + ], + "repeat": "Measurement", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "$Measurement", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "http_req_duration + http_req_blocked", + "value": [ + "http_req_duration", + "http_req_blocked" + ] + }, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "http_req_duration", + "value": "http_req_duration" + }, + { + "selected": true, + "text": "http_req_blocked", + "value": "http_req_blocked" + }, + { + "selected": false, + "text": "http_req_connecting", + "value": "http_req_connecting" + }, + { + "selected": false, + "text": "http_req_looking_up", + "value": "http_req_looking_up" + }, + { + "selected": false, + "text": "http_req_receiving", + "value": "http_req_receiving" + }, + { + "selected": false, + "text": "http_req_sending", + "value": "http_req_sending" + }, + { + "selected": false, + "text": "http_req_waiting", + "value": "http_req_waiting" + } + ], + "query": "http_req_duration,http_req_blocked,http_req_connecting,http_req_looking_up,http_req_receiving,http_req_sending,http_req_waiting", + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Recognize k6 Load Testing Results", + "version": 22 +} \ No newline at end of file diff --git a/load-tests/grafana/provisioning/dashboards/verify-k6-load-testing-results-by-groups.json b/load-tests/grafana/provisioning/dashboards/verify-k6-load-testing-results-by-groups.json new file mode 100644 index 0000000000..cbb5aa9d5a --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/verify-k6-load-testing-results-by-groups.json @@ -0,0 +1,3018 @@ +{ + "__inputs": [ + { + "name": "verify_influxdb_datasource", + "label": "Verify k6 Load Testing Results", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.3.6" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "table-old", + "name": "Table (old)", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool by groups, using the InfluxDB exporter.Based on https://grafana.com/dashboards/10660", + "editable": true, + "gnetId": 13719, + "graphTooltip": 2, + "id": null, + "iteration": 1611037943041, + "links": [], + "panels": [ + { + "collapsed": true, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 68, + "panels": [ + { + "content": "### To choose which measurements to show, use the **Measurements** drop-down in the top left corner!\n\nK6 collects metrics for several phases during the HTTP request lifetime.\n\nThe combined delays for the application and network (send, wait, recieve) are available in this metric:\n * **http_req_duration** _( Total time for request, excluding time spent blocked (http_req_blocked), DNS lookup (http_req_looking_up) and TCP connect (http_req_connecting) time. )_\n\nThis metric can also be broken down into these metrics:\n * **http_req_tls_handshaking** _( Time spent handshaking TLS session with remote host. )_\n * **http_req_sending** _( Time spent sending data to remote host. )_\n * **http_req_waiting** _( Time spent waiting for response from remote host (a.k.a. \"time to first byte\", or \"TTFB\"). )_\n * **http_req_receiving** _( Time spent receiving response data from remote host. )_\n\nThese metrics measure delays and durations on the client:\n\n * **http_req_blocked** _( Time spent blocked (waiting for a free TCP connection slot) before initiating request. )_\n * **http_req_looking_up** _( Time spent looking up remote host name in DNS. )_\n * **http_req_connecting** _( Time spent establishing TCP connection to remote host. )_\n\n\n### View official K6.io documentation here: [https://docs.k6.io/docs/result-metrics][1]\n\n\n[1]: https://docs.k6.io/docs/result-metrics\n\n", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 66, + "links": [], + "mode": "markdown", + "title": "", + "transparent": true, + "type": "text" + } + ], + "repeat": null, + "title": "Documentation", + "type": "row" + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 69, + "panels": [], + "repeat": null, + "title": "K6 Test overview (all URLs)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1157", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 2 + }, + "hiddenSeries": false, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_reqs", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1009", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1010", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:745", + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"errors\" WHERE $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:752", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:753", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 2 + }, + "hiddenSeries": false, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1522", + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1529", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1530", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 70, + "panels": [], + "repeat": "Measurement", + "title": "$Measurement", + "type": "row" + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "decimals": null, + "description": "Grouped by 1 sec intervals and group name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "height": "250px", + "hiddenSeries": false, + "id": 71, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "group" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time per group)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2444", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2445", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "decimals": null, + "description": "Grouped by 1 sec intervals and group name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "height": "250px", + "hiddenSeries": false, + "id": 74, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "name" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time per tag)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2444", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2445", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "min": "#f29191", + "p90": "#629e51", + "p95": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "description": "Grouped by 1 sec intervals", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "height": "250px", + "hiddenSeries": false, + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1318", + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1319", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "heatmap": {}, + "height": "250px", + "hideZeroBuckets": false, + "highlightCards": true, + "id": 8, + "interval": ">1s", + "legend": { + "show": false + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 29 + }, + "height": "50px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (count)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 29 + }, + "height": "50px", + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (min)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 29 + }, + "height": "50px", + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (med)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 29 + }, + "height": "50px", + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (max)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 29 + }, + "height": "50px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "200,500,1000", + "title": "$Measurement (mean)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 29 + }, + "height": "50px", + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=~", + "value": "/^$URL$/" + }, + { + "condition": "AND", + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "thresholds": "", + "title": "$Measurement (p95)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [], + "datasource": "verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 19, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 67, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "P95", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "percentile", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkUrl": "", + "pattern": "url", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "min", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "max", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "mean", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "median", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "min", + "dsType": "influxdb", + "expr": "", + "format": "table", + "groupBy": [ + { + "params": [ + "group" + ], + "type": "tag" + } + ], + "hide": false, + "intervalFactor": 2, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "URLs and latencies (By Group)", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": "verify_influxdb_datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 19, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 73, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "P95", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "percentile", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkUrl": "", + "pattern": "url", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "min", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "max", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "mean", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "median", + "thresholds": [ + "200", + "500" + ], + "type": "number", + "unit": "ms" + }, + { + "alias": "URL", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "min", + "dsType": "influxdb", + "expr": "", + "format": "table", + "groupBy": [ + { + "params": [ + "name" + ], + "type": "tag" + } + ], + "hide": false, + "intervalFactor": 2, + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "median" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [ + { + "key": "group", + "operator": "=~", + "value": "/^$Group$/" + }, + { + "condition": "AND", + "key": "name", + "operator": "=~", + "value": "/^$Tag$/" + } + ] + } + ], + "title": "URLs and latencies (By Name)", + "transform": "table", + "type": "table-old" + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "k6", + "load testing" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "http_req_duration", + "value": [ + "http_req_duration" + ] + }, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "http_req_duration", + "value": "http_req_duration" + }, + { + "selected": false, + "text": "http_req_blocked", + "value": "http_req_blocked" + }, + { + "selected": false, + "text": "http_req_connecting", + "value": "http_req_connecting" + }, + { + "selected": false, + "text": "http_req_looking_up", + "value": "http_req_looking_up" + }, + { + "selected": false, + "text": "http_req_receiving", + "value": "http_req_receiving" + }, + { + "selected": false, + "text": "http_req_sending", + "value": "http_req_sending" + }, + { + "selected": false, + "text": "http_req_waiting", + "value": "http_req_waiting" + } + ], + "query": "http_req_duration,http_req_blocked,http_req_connecting,http_req_looking_up,http_req_receiving,http_req_sending,http_req_waiting", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": "*", + "current": {}, + "datasource": "verify_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "URL", + "multi": false, + "name": "URL", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"name\"", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "verify_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Group", + "multi": true, + "name": "Group", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"group\" WHERE $timeFilter", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "verify_influxdb_datasource", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Tag", + "multi": true, + "name": "Tag", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"name\" WHERE $timeFilter", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "5m", + "30m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Verify k6 Load Testing Results By Groups", + "uid": "XKhgaUpik", + "version": 38 +} \ No newline at end of file diff --git a/load-tests/grafana/provisioning/dashboards/verify-k6-load-testing-results.json b/load-tests/grafana/provisioning/dashboards/verify-k6-load-testing-results.json new file mode 100644 index 0000000000..ef2ba12c6d --- /dev/null +++ b/load-tests/grafana/provisioning/dashboards/verify-k6-load-testing-results.json @@ -0,0 +1,1617 @@ +{ + "__inputs": [ + { + "name": "verify_influxdb_datasource", + "label": "Verify k6 Load Testing Results", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool, using the InfluxDB exporter", + "editable": true, + "gnetId": 2587, + "graphTooltip": 2, + "hideControls": false, + "id": null, + "links": [], + "refresh": "5s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "fill": 1, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "fill": 1, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_reqs", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "fill": 1, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "fill": 1, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (mean)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (max)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT median(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (med)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (min)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (p90)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "verify_influxdb_datasource", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "50px", + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "$Measurement (p95)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "verify_influxdb_datasource", + "description": "Grouped by 1 sec intervals", + "fill": 1, + "height": "250px", + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "verify_influxdb_datasource", + "heatmap": {}, + "height": "250px", + "highlightCards": true, + "id": 8, + "interval": ">1s", + "links": [], + "span": 6, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_req_duration", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"value\" FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketNumber": null, + "yBucketSize": null + } + ], + "repeat": "Measurement", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "$Measurement", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "http_req_duration + http_req_blocked", + "value": [ + "http_req_duration", + "http_req_blocked" + ] + }, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "http_req_duration", + "value": "http_req_duration" + }, + { + "selected": true, + "text": "http_req_blocked", + "value": "http_req_blocked" + }, + { + "selected": false, + "text": "http_req_connecting", + "value": "http_req_connecting" + }, + { + "selected": false, + "text": "http_req_looking_up", + "value": "http_req_looking_up" + }, + { + "selected": false, + "text": "http_req_receiving", + "value": "http_req_receiving" + }, + { + "selected": false, + "text": "http_req_sending", + "value": "http_req_sending" + }, + { + "selected": false, + "text": "http_req_waiting", + "value": "http_req_waiting" + } + ], + "query": "http_req_duration,http_req_blocked,http_req_connecting,http_req_looking_up,http_req_receiving,http_req_sending,http_req_waiting", + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Verify k6 Load Testing Results", + "version": 22 +} \ No newline at end of file diff --git a/load-tests/grafana/provisioning/datasources/datasource.yml b/load-tests/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000000..9a68264bbc --- /dev/null +++ b/load-tests/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,31 @@ +apiVersion: 1 + +datasources: + - name: recognize_influxdb_datasource + type: influxdb + access: proxy + database: recognize + orgId: 1 + url: http://influxdb:8086 + isDefault: true + - name: detect_influxdb_datasource + type: influxdb + access: proxy + database: detect + orgId: 1 + url: http://influxdb:8086 + isDefault: false + - name: verify_influxdb_datasource + type: influxdb + access: proxy + database: verify + orgId: 1 + url: http://influxdb:8086 + isDefault: false + - name: face_verify_influxdb_datasource + type: influxdb + access: proxy + database: face_verify + orgId: 1 + url: http://influxdb:8086 + isDefault: false diff --git a/ui/.husky/pre-commit b/ui/.husky/pre-commit new file mode 100644 index 0000000000..c144ee1eb9 --- /dev/null +++ b/ui/.husky/pre-commit @@ -0,0 +1,5 @@ +#!/bin/sh +. "$(dirname "$0")/_/husky.sh" + +cd ui +npx pretty-quick --staged --pattern "**/*.{js,ts,json,html,scss}" diff --git a/ui/angular.json b/ui/angular.json index 24c7958543..aee3fc9d89 100644 --- a/ui/angular.json +++ b/ui/angular.json @@ -23,10 +23,17 @@ "polyfills": "src/polyfills.ts", "tsConfig": "tsconfig.app.json", "aot": true, - "assets": ["src/favicon.ico", "src/assets"], - "styles": [ "src/styles.scss"], + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "src/styles.scss" + ], "stylePreprocessorOptions": { - "includePaths": ["src/styles/"] + "includePaths": [ + "src/styles/" + ] }, "scripts": [] }, @@ -149,10 +156,17 @@ "polyfills": "src/polyfills.ts", "tsConfig": "tsconfig.spec.json", "karmaConfig": "karma.conf.js", - "assets": ["src/favicon.ico", "src/assets"], - "styles": ["src/styles.scss"], + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "src/styles.scss" + ], "stylePreprocessorOptions": { - "includePaths": ["src/styles/"] + "includePaths": [ + "src/styles/" + ] }, "scripts": [] } @@ -160,7 +174,10 @@ "lint": { "builder": "@angular-eslint/builder:lint", "options": { - "lintFilePatterns": ["src/**/*.ts", "src/**/*.html"] + "lintFilePatterns": [ + "src/**/*.ts", + "src/**/*.html" + ] } }, "e2e": { @@ -180,6 +197,7 @@ }, "defaultProject": "compreface", "cli": { - "defaultCollection": "@ngrx/schematics" + "defaultCollection": "@ngrx/schematics", + "analytics": false } -} +} \ No newline at end of file diff --git a/ui/docker-prod/Dockerfile b/ui/docker-prod/Dockerfile index 51ede6f49b..50345f56f4 100644 --- a/ui/docker-prod/Dockerfile +++ b/ui/docker-prod/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12.7-alpine AS build +FROM node:12.7 AS build WORKDIR /usr/src/app LABEL intermidiate_frs=true COPY . . diff --git a/ui/nginx/conf.d/nginx.conf b/ui/nginx/conf.d/nginx.conf deleted file mode 100644 index f5302c5ada..0000000000 --- a/ui/nginx/conf.d/nginx.conf +++ /dev/null @@ -1,46 +0,0 @@ -upstream frsadmin { - server compreface-admin:8080 fail_timeout=10s max_fails=5; -} - -upstream frsapi { - server compreface-api:8080 fail_timeout=10s max_fails=5; -} - -server { - listen 80; - server_name ui; - - client_max_body_size 5000K; - - location / { - root /usr/share/nginx/html/; - index index.html; - try_files $uri $uri/ /index.html =404; - } - - location /admin/ { - proxy_pass http://frsadmin/; - } - - location /swagger/ { - proxy_pass http://frsapi/; - } - - location /api/v1/ { - if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key'; - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Type' 'text/plain charset=UTF-8'; - add_header 'Content-Length' 0; - return 204; - } - - add_header 'Access-Control-Allow-Origin' '*' always; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key' always; - - proxy_pass http://frsapi/api/v1/; - } -} diff --git a/ui/nginx/templates/nginx.conf.template b/ui/nginx/templates/nginx.conf.template new file mode 100644 index 0000000000..29165e7737 --- /dev/null +++ b/ui/nginx/templates/nginx.conf.template @@ -0,0 +1,74 @@ +upstream frsadmin { + server compreface-admin:8080 fail_timeout=10s max_fails=5; +} + +upstream frsapi { + server compreface-api:8080 fail_timeout=10s max_fails=5; +} + +upstream frscore { + server compreface-core:3000 fail_timeout=10s max_fails=5; +} + +server { + + listen 80; + server_name ui; + + client_max_body_size ${CLIENT_MAX_BODY_SIZE}; + + location / { + root /usr/share/nginx/html/; + index index.html; + try_files $uri $uri/ /index.html =404; + } + + location /admin/ { + proxy_pass http://frsadmin/admin/; + } + + location /api/v1/ { + + proxy_read_timeout ${PROXY_READ_TIMEOUT}; + proxy_connect_timeout ${PROXY_CONNECT_TIMEOUT}; + + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key' always; + + proxy_pass http://frsapi/api/v1/; + } + + location /core/ { + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,x-api-key' always; + + proxy_pass http://frscore/; + } + + location ~ ^/(api|admin)/(swagger-ui.html|webjars|swagger-resources|v2/api-docs)(.*) { + proxy_set_header 'Host' $http_host; + proxy_pass http://frs$1/$2$3$is_args$args; + } +} diff --git a/ui/package.json b/ui/package.json index 8fa7079b1c..725b5e2ce1 100644 --- a/ui/package.json +++ b/ui/package.json @@ -12,7 +12,8 @@ "lint:fix": "ng lint --fix", "e2e": "ng e2e", "postinstall": "ngcc", - "pretty": "prettier --write \"./src/**/*.{js,ts,json,html}\"" + "pretty": "prettier --write \"./src/**/*.{js,ts,json,html,scss}\"", + "prepare": "cd .. && husky install ui/.husky" }, "private": true, "dependencies": { @@ -26,6 +27,8 @@ "@angular/platform-browser": "~11.0.5", "@angular/platform-browser-dynamic": "~11.0.5", "@angular/router": "~11.0.5", + "@ng-web-apis/common": "^2.0.1", + "@ng-web-apis/intersection-observer": "^2.1.0", "@ngrx/data": "^10.0.1", "@ngrx/effects": "^10.0.1", "@ngrx/entity": "^10.0.1", @@ -35,7 +38,10 @@ "@ngrx/store-devtools": "^10.0.1", "@ngx-translate/core": "^13.0.0", "@ngx-translate/http-loader": "^6.0.0", + "chart.js": "^2.9.3", "jasmine-marbles": "^0.6.0", + "ng2-charts": "^2.4.3", + "ngx-infinite-scroll": "^10.0.1", "rxjs": "~6.6.3", "tslib": "^2.0.0", "zone.js": "~0.10.2" @@ -64,9 +70,10 @@ "eslint-plugin-prefer-arrow": "1.2.2", "express": "^4.16.4", "formidable": "^1.2.1", + "husky": "^7.0.2", "jasmine-core": "~3.6.0", "jasmine-spec-reporter": "~5.0.0", - "karma": "~5.1.1", + "karma": "~6.3.14", "karma-chrome-launcher": "~3.1.0", "karma-coverage-istanbul-reporter": "~3.0.2", "karma-jasmine": "~4.0.0", @@ -74,6 +81,7 @@ "nodemod": "^2.2.3", "prettier": "^2.1.1", "prettier-eslint": "^12.0.0", + "pretty-quick": "^3.1.1", "protractor": "~7.0.0", "ts-node": "~7.0.0", "typescript": "~4.0.3" diff --git a/ui/proxy.conf.json b/ui/proxy.conf.json index 149e2c8e62..b920132cdb 100644 --- a/ui/proxy.conf.json +++ b/ui/proxy.conf.json @@ -10,5 +10,11 @@ "secure": false, "logLevel": "debug", "changeOrigin": true + }, + "/core/*": { + "target": "http://localhost:8000", + "secure": false, + "logLevel": "debug", + "changeOrigin": true } } diff --git a/ui/src/app/app-routing.module.ts b/ui/src/app/app-routing.module.ts index 8a50009e2b..a5596b06aa 100644 --- a/ui/src/app/app-routing.module.ts +++ b/ui/src/app/app-routing.module.ts @@ -19,6 +19,8 @@ import { RouterModule, Routes } from '@angular/router'; import { MainLayoutComponent } from './ui/main-layout/main-layout.component'; import { DemoLayoutComponent } from './ui/demo-layout/demo-layout.component'; import { UserInfoResolver } from './core/user-info/user-info.resolver'; +import { CreateApplicationGuard } from './pages/create-application-wizard/create-application.guard'; +import { PasswordRecoveryComponent } from './pages/password-recovery/password-recovery.component'; const routes: Routes = [ { @@ -39,6 +41,19 @@ const routes: Routes = [ resolve: [UserInfoResolver], children: [{ path: '', loadChildren: () => import('./pages/test-model/test-model.module').then(m => m.TestModelModule) }], }, + { + path: 'create-application', + component: MainLayoutComponent, + canActivate: [CreateApplicationGuard], + resolve: [UserInfoResolver], + children: [ + { + path: '', + loadChildren: () => + import('./pages/create-application-wizard/create-application-wizard.module').then(m => m.CreateApplicationWizardModule), + }, + ], + }, { path: 'manage-collection', component: MainLayoutComponent, @@ -47,11 +62,26 @@ const routes: Routes = [ { path: '', loadChildren: () => import('./pages/manage-collection/manage-collection.module').then(m => m.ManageCollectionModule) }, ], }, + { + path: 'dashboard', + component: MainLayoutComponent, + resolve: [UserInfoResolver], + children: [ + { path: '', loadChildren: () => import('./pages/model-dashboard/model-dashboard.module').then(m => m.ModelDashboardModule) }, + ], + }, { path: 'demo', component: DemoLayoutComponent, children: [{ path: '', loadChildren: () => import('./pages/demo/demo.module').then(m => m.DemoModule) }], }, + { + path: 'reset-password', + component: PasswordRecoveryComponent, + children: [ + { path: '', loadChildren: () => import('./pages/password-recovery/password-recovery.module').then(m => m.PasswordRecoveryModule) }, + ], + }, { path: 'login', loadChildren: () => import('./pages/login/login.module').then(m => m.LoginModule) }, { path: 'sign-up', loadChildren: () => import('./pages/sign-up/sign-up.module').then(m => m.SignUpModule) }, { path: '**', redirectTo: '/' }, diff --git a/ui/src/app/app.component.html b/ui/src/app/app.component.html index 8c640427c7..c13c794876 100644 --- a/ui/src/app/app.component.html +++ b/ui/src/app/app.component.html @@ -14,4 +14,10 @@ ~ permissions and limitations under the License. --> - + + + + + + + diff --git a/ui/src/app/app.component.scss b/ui/src/app/app.component.scss index 438348484a..249c031f13 100644 --- a/ui/src/app/app.component.scss +++ b/ui/src/app/app.component.scss @@ -13,4 +13,3 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ - diff --git a/ui/src/app/app.component.ts b/ui/src/app/app.component.ts index 265d7bc20e..a0af65e14a 100644 --- a/ui/src/app/app.component.ts +++ b/ui/src/app/app.component.ts @@ -13,27 +13,81 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { Component } from '@angular/core'; +import { Component, OnInit } from '@angular/core'; import { Store } from '@ngrx/store'; import { TranslateService } from '@ngx-translate/core'; import { AuthService } from './core/auth/auth.service'; import { AppState } from './store'; import { CustomIconsService } from './core/custom-icons/custom-icons.service'; +import { getMaxImageSize } from './store/image-size/actions'; +import { refreshToken } from './store/auth/action'; +import { GranTypes } from './data/enums/gran_type.enum'; +import { selectUserId } from './store/userInfo/selectors'; +import { Observable, Subscription } from 'rxjs'; +import { filter, tap } from 'rxjs/operators'; +import { getPlugin } from './store/landmarks-plugin/action'; +import { getMailServiceStatus } from './store/mail-service/actions'; +import { getBeServerStatus, getCoreServerStatus, getDbServerStatus } from './store/servers-status/actions'; +import { selectServerStatus } from './store/servers-status/selectors'; +import { ServerStatusInt } from './store/servers-status/reducers'; @Component({ selector: 'app-root', templateUrl: './app.component.html', styleUrls: ['./app.component.scss'], }) -export class AppComponent { +export class AppComponent implements OnInit { + userId$: Observable; + serverStatus$: Observable; + + serverStatus: ServerStatusInt; + subs: Subscription; + constructor( auth: AuthService, - store: Store, + private store: Store, private translate: TranslateService, private customIconsService: CustomIconsService ) { translate.setDefaultLang('en'); customIconsService.registerIcons(); + this.userId$ = this.store.select(selectUserId); + this.serverStatus$ = this.store.select(selectServerStatus).pipe(filter(status => !!status)); + } + + ngOnInit(): void { + this.store.dispatch(getBeServerStatus({ preserveState: false })); + this.store.dispatch(getDbServerStatus({ preserveState: false })); + this.store.dispatch(getCoreServerStatus({ preserveState: false })); + + this.subs = this.serverStatus$.subscribe(status => { + this.serverStatus = status; + if (status.coreStatus && status.apiStatus && status.status) { + this.getUserId(); + } + }); + } + + getUserId(): void { + const subs = this.userId$ + .pipe( + filter(userId => !!userId), + tap(() => { + this.store.dispatch(getMaxImageSize()); + this.store.dispatch(getPlugin()); + const payload = { + grant_type: GranTypes.RefreshToken, + scope: 'all', + }; + this.store.dispatch(getMailServiceStatus()); + setInterval(() => this.store.dispatch(refreshToken(payload)), 300000); + }) + ) + .subscribe(() => subs.unsubscribe()); + } + + ngOnDestroy(): void { + this.subs.unsubscribe(); } } diff --git a/ui/src/app/app.module.ts b/ui/src/app/app.module.ts index 67388d862e..0e62f10cc8 100644 --- a/ui/src/app/app.module.ts +++ b/ui/src/app/app.module.ts @@ -30,7 +30,6 @@ import { MatToolbarModule } from '@angular/material/toolbar'; import { BrowserModule } from '@angular/platform-browser'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { TranslateLoader, TranslateModule } from '@ngx-translate/core'; -import { TranslateHttpLoader } from '@ngx-translate/http-loader'; import { CreateDialogComponent } from 'src/app/features/create-dialog/create-dialog.component'; import { AppRoutingModule } from './app-routing.module'; @@ -51,6 +50,12 @@ import { UserInfoResolver } from './core/user-info/user-info.resolver'; import { RoleEditDialogComponent } from './features/role-edit-dialog/role-edit-dialog.component'; import { MatSelectModule } from '@angular/material/select'; import { MergerDialogComponent } from './features/merger-dialog/merger-dialog.component'; +import { MatExpansionModule } from '@angular/material/expansion'; +import { EditSubjectDialog } from './features/edit-subject/edit-subject-dialog.component'; +import { SideMenuModule } from './features/side-menu/side-menu.module'; +import { SpinnerModule } from './features/spinner/spinner.module'; +import { ServerStatusComponent } from './pages/server-status/server-status.component'; +import { NoCacheTranslateLoader } from './no-cache-translate-loader'; @NgModule({ declarations: [ @@ -59,10 +64,12 @@ import { MergerDialogComponent } from './features/merger-dialog/merger-dialog.co DemoLayoutComponent, CreateDialogComponent, EditDialogComponent, + EditSubjectDialog, AlertComponent, DeleteDialogComponent, RoleEditDialogComponent, MergerDialogComponent, + ServerStatusComponent, ], imports: [ BrowserModule, @@ -81,16 +88,19 @@ import { MergerDialogComponent } from './features/merger-dialog/merger-dialog.co FormsModule, ToolBarModule, FooterModule, + SideMenuModule, AppStoreModule, HttpClientModule, SnackBarModule, MatRadioModule, + MatExpansionModule, BreadcrumbsModule, + SpinnerModule, BreadcrumbsContainerModule, TranslateModule.forRoot({ loader: { provide: TranslateLoader, - useFactory: (http: HttpClient) => new TranslateHttpLoader(http), + useFactory: (http: HttpClient) => new NoCacheTranslateLoader(http), deps: [HttpClient], }, }), @@ -107,6 +117,13 @@ import { MergerDialogComponent } from './features/merger-dialog/merger-dialog.co ], bootstrap: [AppComponent], exports: [CreateDialogComponent], - entryComponents: [CreateDialogComponent, AlertComponent, EditDialogComponent, DeleteDialogComponent, RoleEditDialogComponent], + entryComponents: [ + CreateDialogComponent, + AlertComponent, + EditDialogComponent, + EditSubjectDialog, + DeleteDialogComponent, + RoleEditDialogComponent, + ], }) export class AppModule {} diff --git a/ui/src/app/core/auth/auth.service.spec.ts b/ui/src/app/core/auth/auth.service.spec.ts index 997683c989..8d34a35fbf 100644 --- a/ui/src/app/core/auth/auth.service.spec.ts +++ b/ui/src/app/core/auth/auth.service.spec.ts @@ -67,7 +67,7 @@ describe('AuthService', () => { password: 'password', }; - service.logIn(dummyUser.firstName, dummyUser.password).subscribe(); + service.logIn(dummyUser.firstName, dummyUser.password, 'password').subscribe(); const request = httpMock.expectOne(`${environment.adminApiUrl}${API.Login}`); expect(request.request.method).toBe('POST'); diff --git a/ui/src/app/core/auth/auth.service.ts b/ui/src/app/core/auth/auth.service.ts index c963da9c93..84fa3e2591 100644 --- a/ui/src/app/core/auth/auth.service.ts +++ b/ui/src/app/core/auth/auth.service.ts @@ -25,6 +25,8 @@ import { API } from '../../data/enums/api-url.enum'; import { Routes } from '../../data/enums/routers-url.enum'; import { AppState } from '../../store'; import { SignUp } from '../../data/interfaces/sign-up'; +import { shareReplay, tap } from 'rxjs/operators'; +import { selectUserId } from 'src/app/store/userInfo/selectors'; @Injectable({ providedIn: 'root', @@ -32,16 +34,19 @@ import { SignUp } from '../../data/interfaces/sign-up'; export class AuthService { refreshInProgress: boolean; requests = []; + currentUserId$: Observable; - constructor(private http: HttpClient, private formBuilder: FormBuilder, private store: Store, private router: Router) {} + constructor(private http: HttpClient, private formBuilder: FormBuilder, private store: Store, private router: Router) { + this.currentUserId$ = this.store.select(selectUserId).pipe(shareReplay(1)); + } - logIn(email: string, password: string): Observable { + logIn(email: string, password: string, grant_type: string): Observable { const url = `${environment.adminApiUrl}${API.Login}`; const form = this.formBuilder.group({ email, password, // eslint-disable-next-line @typescript-eslint/naming-convention - grant_type: 'password', + grant_type, }); const formData = new FormData(); formData.append('username', form.get('email').value); @@ -52,6 +57,18 @@ export class AuthService { return this.http.post(url, formData, { headers: { Authorization: environment.basicToken }, withCredentials: false }); } + refreshToken(grant_type: string): Observable { + const url = `${environment.adminApiUrl}${API.Login}?grant_type=${grant_type}&scope=all`; + const form = this.formBuilder.group({ + scope: 'all', + grant_type, + }); + const formData = new FormData(); + formData.append('grant_type', form.get('grant_type').value); + formData.append('scope', form.get('scope').value); + return this.http.post(url, formData, { headers: { Authorization: environment.basicToken }, withCredentials: false }); + } + clearUserToken(): Observable { const url = `${environment.adminApiUrl}${API.Login}`; @@ -77,8 +94,23 @@ export class AuthService { this.router.navigate([Routes.Login], { ...queryParam }); } + navigateToLogin(): void { + this.router.navigate([Routes.Login]); + } + changePassword(oldPassword: string, newPassword: string): Observable { const url = `${environment.adminApiUrl}${API.ChangePassword}`; return this.http.put(url, { oldPassword, newPassword }, { observe: 'response' }); } + + recoveryPassword(email: string): Observable { + const url = `${environment.adminApiUrl}${API.ForgotPassword}`; + return this.http.post(url, { email: email }); + } + + updatePassword(password: string, token: string): Observable { + const url = `${environment.adminApiUrl}${API.ResetPassword}?token=${token}`; + + return this.http.put(url, { password: password }, { observe: 'response' }); + } } diff --git a/ui/src/app/core/auth/error.inerceptor.ts b/ui/src/app/core/auth/error.inerceptor.ts index bee7aecc35..4461656ed2 100644 --- a/ui/src/app/core/auth/error.inerceptor.ts +++ b/ui/src/app/core/auth/error.inerceptor.ts @@ -15,28 +15,57 @@ */ import { HttpErrorResponse, HttpEvent, HttpHandler, HttpInterceptor, HttpRequest } from '@angular/common/http'; import { Injectable, Injector } from '@angular/core'; -import { Observable, throwError } from 'rxjs'; -import { catchError } from 'rxjs/operators'; +import { Observable, combineLatest, throwError } from 'rxjs'; +import { catchError, filter, take, tap } from 'rxjs/operators'; import { AuthService } from './auth.service'; +import { Store } from '@ngrx/store'; +import { AppState } from 'src/app/store'; +import { getBeServerStatus, getCoreServerStatus, getDbServerStatus } from 'src/app/store/servers-status/actions'; +import { ServerStatusInt } from 'src/app/store/servers-status/reducers'; +import { selectServerStatus } from 'src/app/store/servers-status/selectors'; @Injectable() export class ErrorInterceptor implements HttpInterceptor { private authService: AuthService; + serverStatus$: Observable; - constructor(private injector: Injector) { + constructor(private injector: Injector, private store: Store) { this.authService = this.injector.get(AuthService); + this.serverStatus$ = this.store.select(selectServerStatus).pipe(filter(status => !!status)); } intercept(request: HttpRequest, next: HttpHandler): Observable> { return next.handle(request).pipe( catchError((response: any): Observable> => { - if (response instanceof HttpErrorResponse && response.status === 401) { - this.authService.logOut(); + if (response instanceof HttpErrorResponse) { + if (response.status === 401) { + this.authService.logOut(); + } else if (response.status === 502) { + combineLatest([this.authService.currentUserId$, this.serverStatus$]) + .pipe( + take(1), + tap(([userId, statuses]) => { + const { status, apiStatus, coreStatus } = statuses; + if (!userId) { + this.authService.navigateToLogin(); + } else { + this.updateServerStatus(apiStatus, status, coreStatus); + } + }) + ) + .subscribe(); + } } - return throwError(response); }) ); } + + private updateServerStatus(apiStatus: string, status: string, coreStatus: string): void { + const preserveState = !(apiStatus && status && coreStatus); + this.store.dispatch(getBeServerStatus({preserveState})); + this.store.dispatch(getDbServerStatus({preserveState})); + this.store.dispatch(getCoreServerStatus({preserveState})); + } } diff --git a/java/admin/src/main/java/com/exadel/frs/dto/AccessToken.java b/ui/src/app/core/collection/collection.helper.ts similarity index 69% rename from java/admin/src/main/java/com/exadel/frs/dto/AccessToken.java rename to ui/src/app/core/collection/collection.helper.ts index 85ab5d0b66..bc8695bf4c 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/AccessToken.java +++ b/ui/src/app/core/collection/collection.helper.ts @@ -1,29 +1,23 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.dto; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.AllArgsConstructor; -import lombok.Data; - -@Data -@AllArgsConstructor -public class AccessToken { - - @JsonProperty("access_token") - private String accessToken; -} \ No newline at end of file +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { environment } from 'src/environments/environment'; + +export class CollectionHelper { + static getCollectionItemUrl(apiKey: string, imageId: string): string { + return `${environment.userApiUrl}static/${apiKey}/images/${imageId}`; + } +} diff --git a/ui/src/app/core/collection/collection.service.ts b/ui/src/app/core/collection/collection.service.ts index d639420f94..c71a7db55c 100644 --- a/ui/src/app/core/collection/collection.service.ts +++ b/ui/src/app/core/collection/collection.service.ts @@ -18,6 +18,9 @@ import { HttpClient } from '@angular/common/http'; import { Observable } from 'rxjs'; import { environment } from '../../../environments/environment'; +import { CollectionItem, SubjectExampleResponseItem } from 'src/app/data/interfaces/collection'; +import { map } from 'rxjs/operators'; +import { CollectionInfo } from 'src/app/data/interfaces/collection-info'; @Injectable({ providedIn: 'root', @@ -38,8 +41,9 @@ export class CollectionService { } editSubject(editName: string, apiKey: string, subject: string): Observable<{ updated: boolean }> { + const subjectEncoded = encodeURIComponent(subject); return this.http.put<{ updated: boolean }>( - `${environment.userApiUrl}recognition/subjects/${subject}`, + `${environment.userApiUrl}recognition/subjects/${subjectEncoded}`, { subject: editName }, { headers: { 'x-api-key': apiKey }, @@ -48,7 +52,59 @@ export class CollectionService { } deleteSubject(subject: string, apiKey: string): Observable<{ subject: string }> { - return this.http.delete<{ subject: string }>(`${environment.userApiUrl}recognition/subjects/${subject}`, { + const subjectEncoded = encodeURIComponent(subject); + return this.http.delete<{ subject: string }>(`${environment.userApiUrl}recognition/subjects/${subjectEncoded}`, { + headers: { 'x-api-key': apiKey }, + }); + } + + getSubjectMediaNextPage(apiKey: string, subject: string, next: number): Observable { + const pageSize = 15; + return this.getSubjectMedia(apiKey, subject, next, pageSize); + } + + getSubjectMedia(apiKey: string, subject: string, page: number = 0, size: number = 15) { + const subjectEncoded = encodeURIComponent(subject); + return this.http + .get(`${environment.userApiUrl}recognition/faces?size=${size}&subject=${subjectEncoded}&page=${page}`, { + headers: { 'x-api-key': apiKey }, + }) + .pipe( + map((resp: CollectionInfo) => { + const totalPages = resp.total_pages; + const totalElements = resp.total_elements; + return resp.faces.map(el => ({ ...el, page: page, totalPages: totalPages, totalElements: totalElements })); + }) + ); + } + + getTotalImagesInfo(apiKey: string): Observable { + return this.http + .get(`${environment.userApiUrl}recognition/faces`, { + headers: { 'x-api-key': apiKey }, + }) + .pipe(map((resp: CollectionInfo) => resp.total_elements)); + } + + uploadSubjectExamples(item: CollectionItem, subject: string, apiKey: string): Observable { + const { file } = item; + const formData = new FormData(); + formData.append('file', file, file.name); + const subjectEncoded = encodeURIComponent(subject); + + return this.http.post(`${environment.userApiUrl}recognition/faces?subject=${subjectEncoded}`, formData, { + headers: { 'x-api-key': apiKey }, + }); + } + + deleteSubjectExample(item: CollectionItem, apiKey: string): Observable { + return this.http.delete(`${environment.userApiUrl}recognition/faces/${item.id}`, { + headers: { 'x-api-key': apiKey }, + }); + } + + deleteSubjectExamplesBulk(ids: string[], apiKey: string): Observable { + return this.http.post(`${environment.userApiUrl}recognition/faces/delete`, ids, { headers: { 'x-api-key': apiKey }, }); } diff --git a/ui/src/app/core/constants.ts b/ui/src/app/core/constants.ts index 3eb93742c8..d72acaa349 100644 --- a/ui/src/app/core/constants.ts +++ b/ui/src/app/core/constants.ts @@ -15,5 +15,5 @@ */ export const EMAIL_REGEXP_PATTERN = '^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$'; -export const MAX_IMAGE_SIZE = 5000000; +export const MAX_INPUT_LENGTH = 50; export const AVAILABLE_IMAGE_EXTENSIONS: string[] = ['jpeg', 'jpg', 'png', 'webp']; diff --git a/ui/src/app/core/custom-icons/custom-icons.ts b/ui/src/app/core/custom-icons/custom-icons.ts index c48d555ad3..eb25c5e420 100644 --- a/ui/src/app/core/custom-icons/custom-icons.ts +++ b/ui/src/app/core/custom-icons/custom-icons.ts @@ -18,16 +18,30 @@ export enum Icons { Avatar = 'avatar-subject', Add = 'add_new', + Alert = 'alert', BodyScan = 'body-scan', + Axis = 'axis', Close = 'close', Copy = 'copy', + Check = 'check', Cross = 'cross_new', + Dashboard = 'dashboard', + Dashboard_White = 'dashboard-white', Edit = 'edit', + Face = 'face-icon', + Face_white = 'face-icon-white', + Image = 'image-icon', Info = 'info_new', MoreVent = 'more-vert_new', + Play = 'play-icon', + Play_White = 'play-icon-white', Profile = 'profile', Search = 'search', + Search_Subject = 'subject-search', + Settings = 'settings', Trash = 'trash', + Trash_Subject = 'trash-subject', Upload = 'upload', + User_icon = 'user-icon', Warning = 'warning', } diff --git a/ui/src/app/core/face-recognition/face-recognition.service.ts b/ui/src/app/core/face-recognition/face-recognition.service.ts index b811a24a1a..04371eb8a1 100644 --- a/ui/src/app/core/face-recognition/face-recognition.service.ts +++ b/ui/src/app/core/face-recognition/face-recognition.service.ts @@ -31,7 +31,7 @@ import { RequestResultVerification } from '../../data/interfaces/response-result export class FaceRecognitionService { headers: HttpHeaders; - constructor(private http: HttpClient) {} + constructor(private http: HttpClient) { } addFace(file: any, model: Model): Observable { const formData = new FormData(); @@ -42,7 +42,7 @@ export class FaceRecognitionService { }); } - recognize(file: any, apiKey: string): Observable { + recognize(file: any, apiKey: string, landmarks: string): Observable { const url = `${environment.userApiUrl}recognition/recognize`; const formData = new FormData(); formData.append('file', file); @@ -51,7 +51,7 @@ export class FaceRecognitionService { .post(url, formData, { headers: { 'x-api-key': apiKey }, // eslint-disable-next-line @typescript-eslint/naming-convention - params: { face_plugins: ['landmarks', 'gender', 'age'] }, + params: { face_plugins: [landmarks, 'gender', 'age', 'pose'] }, }) .pipe( map(data => ({ @@ -61,7 +61,7 @@ export class FaceRecognitionService { ); } - detection(file: any, apiKey: string): Observable { + detection(file: any, apiKey: string, landmarks: string): Observable { const url = `${environment.userApiUrl}detection/detect`; const formData = new FormData(); formData.append('file', file); @@ -70,7 +70,7 @@ export class FaceRecognitionService { .post(url, formData, { headers: { 'x-api-key': apiKey }, // eslint-disable-next-line @typescript-eslint/naming-convention - params: { face_plugins: ['landmarks', 'gender', 'age'] }, + params: { face_plugins: [landmarks, 'gender', 'age', 'pose'] }, }) .pipe( map(data => ({ @@ -83,7 +83,8 @@ export class FaceRecognitionService { verification( sourceImage: File, targetImage: File, - apiKey: string + apiKey: string, + landmarks: string ): Observable<{ data: { result: RequestResultVerification }; request: string }> { const url = `${environment.userApiUrl}verification/verify`; const formData: FormData = new FormData(); @@ -95,7 +96,7 @@ export class FaceRecognitionService { .post(url, formData, { headers: { 'x-api-key': apiKey }, // eslint-disable-next-line @typescript-eslint/naming-convention - params: { face_plugins: ['landmarks', 'gender', 'age'] }, + params: { face_plugins: [landmarks, 'gender', 'age', 'pose'] }, }) .pipe( map(data => data as { result: RequestResultVerification }), @@ -128,7 +129,7 @@ export class FaceRecognitionService { apiKey, file: { name: fname }, } = options; - return `curl -X POST "${window.location.origin}${url}?face_plugins=[landmarks, gender, age]" \\\n-H "Content-Type: multipart/form-data" \\\n-H "x-api-key: ${apiKey}" \\\n-F "file=@${fname}"`; + return `curl -X POST "${window.location.origin}${url}?face_plugins=landmarks, gender, age, pose" \\\n-H "Content-Type: multipart/form-data" \\\n-H "x-api-key: ${apiKey}" \\\n-F "file=@${fname}"`; } private createUIDoubleFileRequest(url: string, options = {} as UIDoubleFileRequestOptions, params = {}): string { @@ -137,6 +138,6 @@ export class FaceRecognitionService { sourceImage: { name: ffname }, targetImage: { name: sfname }, } = options; - return `curl -X POST "${window.location.origin}${url}?face_plugins=[landmarks, gender, age]" \\\n-H "Content-Type: multipart/form-data" \\\n-H "x-api-key: ${apiKey}" \\\n-F "source_image=@${ffname}" \\\n-F "target_image=@${sfname}"`; + return `curl -X POST "${window.location.origin}${url}?face_plugins=landmarks, gender, age, pose" \\\n-H "Content-Type: multipart/form-data" \\\n-H "x-api-key: ${apiKey}" \\\n-F "source_image=@${ffname}" \\\n-F "target_image=@${sfname}"`; } } diff --git a/ui/src/app/core/image-size/image-size.service.ts b/ui/src/app/core/image-size/image-size.service.ts new file mode 100644 index 0000000000..48eb888a03 --- /dev/null +++ b/ui/src/app/core/image-size/image-size.service.ts @@ -0,0 +1,16 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { Observable } from 'rxjs'; +import { MaxImageSize } from 'src/app/data/interfaces/size.interface'; +import { environment } from 'src/environments/environment'; + +@Injectable({ + providedIn: 'root', +}) +export class ImageSizeService { + constructor(private http: HttpClient) {} + + fetchMaxSize(): Observable { + return this.http.get(`${environment.userApiUrl}config`); + } +} diff --git a/ui/src/app/core/mail-service/mail-service.service.ts b/ui/src/app/core/mail-service/mail-service.service.ts new file mode 100644 index 0000000000..35bb39e150 --- /dev/null +++ b/ui/src/app/core/mail-service/mail-service.service.ts @@ -0,0 +1,19 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { Observable } from 'rxjs'; +import { API } from 'src/app/data/enums/api-url.enum'; +import { MailServiceStatus } from 'src/app/data/interfaces/mail-service-status'; +import { environment } from '../../../environments/environment'; + +@Injectable({ + providedIn: 'root', +}) +export class MailService { + constructor(private http: HttpClient) {} + + getStatus(): Observable { + const url = `${environment.adminApiUrl}${API.MailServiceStatus}`; + + return this.http.get(url); + } +} diff --git a/ui/src/app/core/model/model.service.ts b/ui/src/app/core/model/model.service.ts index 148a63d855..23ec5dc0d3 100644 --- a/ui/src/app/core/model/model.service.ts +++ b/ui/src/app/core/model/model.service.ts @@ -44,12 +44,20 @@ export class ModelService { ); } + getModel(applicationId: string, currentModelId: string): Observable { + return this.http.get(`${environment.adminApiUrl}app/${applicationId}/model/${currentModelId}`); + } + getAll(applicationId: string): Observable { return this.http .get(`${environment.adminApiUrl}app/${applicationId}/models`) .pipe(switchMap(models => this.filterModel(models))); } + getStatistics(appId: string, modelId: string): Observable { + return this.http.get(`${environment.adminApiUrl}app/${appId}/model/${modelId}/statistics`); + } + create(applicationId: string, name: string, type: string): Observable { name = name.trim(); return this.http.post(`${environment.adminApiUrl}app/${applicationId}/model`, { name, type }); diff --git a/ui/src/app/core/photo-loader/photo-loader.service.ts b/ui/src/app/core/photo-loader/photo-loader.service.ts index 6438a83f9b..7f0affa56b 100644 --- a/ui/src/app/core/photo-loader/photo-loader.service.ts +++ b/ui/src/app/core/photo-loader/photo-loader.service.ts @@ -36,4 +36,10 @@ export class LoadingPhotoService { return this.createImage(url); } + + getPlugin(): Observable { + const url: string = '/core/status'; + + return this.http.get(url); + } } diff --git a/ui/src/app/core/server-status/server-status.service.ts b/ui/src/app/core/server-status/server-status.service.ts new file mode 100644 index 0000000000..d7c0e8932f --- /dev/null +++ b/ui/src/app/core/server-status/server-status.service.ts @@ -0,0 +1,18 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { Observable } from 'rxjs'; +import { ServerStatusInt } from 'src/app/store/servers-status/reducers'; +import { environment } from 'src/environments/environment'; + +@Injectable({ + providedIn: 'root', +}) +export class ServerStatusService { + constructor(private http: HttpClient) {} + + getServerStatus(): Observable { + const url = `${environment.adminApiUrl}status`; + + return this.http.get(url); + } +} diff --git a/ui/src/app/core/user-info/user-info.resolver.ts b/ui/src/app/core/user-info/user-info.resolver.ts index 4b69828a36..070ec4ede8 100644 --- a/ui/src/app/core/user-info/user-info.resolver.ts +++ b/ui/src/app/core/user-info/user-info.resolver.ts @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + import { Injectable } from '@angular/core'; import { Resolve } from '@angular/router'; import { Store } from '@ngrx/store'; diff --git a/ui/src/app/data/enums/api-url.enum.ts b/ui/src/app/data/enums/api-url.enum.ts index bcfa89eb73..a67818f8f7 100644 --- a/ui/src/app/data/enums/api-url.enum.ts +++ b/ui/src/app/data/enums/api-url.enum.ts @@ -20,4 +20,7 @@ export enum API { Logout = 'oauth/logout', UserInfo = 'user/me', ChangePassword = 'user/me/password', + ForgotPassword = 'user/forgot-password', + ResetPassword = 'user/reset-password', + MailServiceStatus = 'config', } diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelShareResponseDto.java b/ui/src/app/data/enums/circle-loading-progress.enum.ts similarity index 76% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/ModelShareResponseDto.java rename to ui/src/app/data/enums/circle-loading-progress.enum.ts index 97a5dff09d..f7e8664041 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelShareResponseDto.java +++ b/ui/src/app/data/enums/circle-loading-progress.enum.ts @@ -1,28 +1,22 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.dto.ui; - -import java.util.UUID; -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class ModelShareResponseDto { - - private UUID modelRequestUuid; -} \ No newline at end of file +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +export enum CircleLoadingProgressEnum { + Uploaded = 'uploaded', + Failed = 'failed', + InProgress = 'inprogress', + OnHold = 'onhold', +} diff --git a/ui/src/app/data/enums/gran_type.enum.ts b/ui/src/app/data/enums/gran_type.enum.ts new file mode 100644 index 0000000000..9172c76790 --- /dev/null +++ b/ui/src/app/data/enums/gran_type.enum.ts @@ -0,0 +1,4 @@ +export enum GranTypes { + Password = 'password', + RefreshToken = 'refresh_token', +} diff --git a/ui/src/app/data/enums/routers-url.enum.ts b/ui/src/app/data/enums/routers-url.enum.ts index 11564943bd..fc9937d31b 100644 --- a/ui/src/app/data/enums/routers-url.enum.ts +++ b/ui/src/app/data/enums/routers-url.enum.ts @@ -18,8 +18,12 @@ export enum Routes { Login = '/login', Home = '/', Application = '/application', + CreateApplication = '/create-application', SignUp = '/sign-up', TestModel = '/test-model', ManageCollection = '/manage-collection', + Dashboard = '/dashboard', + AppUsers = '/application-users', Demo = '/demo', + UpdatePassword = '/password-update', } diff --git a/ui/src/app/data/enums/servers-status.ts b/ui/src/app/data/enums/servers-status.ts new file mode 100644 index 0000000000..4ed5f5bb54 --- /dev/null +++ b/ui/src/app/data/enums/servers-status.ts @@ -0,0 +1,3 @@ +export enum ServerStatus { + Ready = 'OK', +} diff --git a/ui/src/app/data/enums/subject-mode.enum.ts b/ui/src/app/data/enums/subject-mode.enum.ts new file mode 100644 index 0000000000..06fa4cfca1 --- /dev/null +++ b/ui/src/app/data/enums/subject-mode.enum.ts @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +export enum SubjectModeEnum { + Default = 'default', + BulkSelect = 'bulk_select', + } + \ No newline at end of file diff --git a/ui/src/app/data/interfaces/collection-info.ts b/ui/src/app/data/interfaces/collection-info.ts new file mode 100644 index 0000000000..a99bdd970d --- /dev/null +++ b/ui/src/app/data/interfaces/collection-info.ts @@ -0,0 +1,9 @@ +import { SubjectExampleResponseItem } from './collection'; + +export interface CollectionInfo { + faces: SubjectExampleResponseItem[]; + page_number: number; + page_side: number; + total_elements: number; + total_pages: number; +} diff --git a/ui/src/app/data/interfaces/collection.ts b/ui/src/app/data/interfaces/collection.ts index 9b67594297..24b1e655a0 100644 --- a/ui/src/app/data/interfaces/collection.ts +++ b/ui/src/app/data/interfaces/collection.ts @@ -13,6 +13,23 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ +import { CircleLoadingProgressEnum } from '../enums/circle-loading-progress.enum'; + export interface Collection { subjects: string[]; } + +export interface CollectionItem { + url: string; + id?: string; + subject: string; + status: CircleLoadingProgressEnum; + file?: File; + error?: string; + isSelected?: boolean; +} + +export interface SubjectExampleResponseItem { + subject: string; + image_id: string; +} diff --git a/ui/src/app/data/interfaces/demo-status.ts b/ui/src/app/data/interfaces/demo-status.ts index 2d53fb8db7..6b678f1b0a 100644 --- a/ui/src/app/data/interfaces/demo-status.ts +++ b/ui/src/app/data/interfaces/demo-status.ts @@ -16,4 +16,6 @@ export interface DemoStatus { demoFaceCollectionIsInconsistent: boolean; saveImagesToDB: boolean; + dbIsInconsistent: boolean; + status: string; } diff --git a/ui/src/app/data/interfaces/face-matches.ts b/ui/src/app/data/interfaces/face-matches.ts index 5356fa0c2a..90c17e89d6 100644 --- a/ui/src/app/data/interfaces/face-matches.ts +++ b/ui/src/app/data/interfaces/face-matches.ts @@ -16,11 +16,13 @@ import { BoxSize } from './box-size'; import { PersonAge } from './person-age'; import { PersonGender } from './person-gender'; +import { PoseSubject } from './pose-subjects'; export interface FaceMatches { age: PersonAge; gender: PersonGender; box: BoxSize; + pose: PoseSubject; similarity: number; landmarks: [number[]]; } diff --git a/ui/src/app/features/user-list/user-list.component.scss b/ui/src/app/data/interfaces/mail-service-status.ts similarity index 87% rename from ui/src/app/features/user-list/user-list.component.scss rename to ui/src/app/data/interfaces/mail-service-status.ts index 454b8312a0..843413c89f 100644 --- a/ui/src/app/features/user-list/user-list.component.scss +++ b/ui/src/app/data/interfaces/mail-service-status.ts @@ -1,24 +1,19 @@ -/*! - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -.user-list { - position: relative; -} - -app-spinner { - position: absolute; - left: 40%; -} +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +export interface MailServiceStatus { + mailServiceEnabled: boolean; +} diff --git a/ui/src/app/data/interfaces/model.ts b/ui/src/app/data/interfaces/model.ts index f18a4186b4..4958b5face 100644 --- a/ui/src/app/data/interfaces/model.ts +++ b/ui/src/app/data/interfaces/model.ts @@ -30,6 +30,9 @@ export interface Model { }; role: string; apiKey?: string; + imageCount?: number; + subjectCount?: number; + createdDate?: Date; } export interface ModelUpdate { @@ -37,4 +40,5 @@ export interface ModelUpdate { applicationId: string; modelId: string; type: string; + isFirstService: boolean } diff --git a/ui/src/app/data/interfaces/plugins.ts b/ui/src/app/data/interfaces/plugins.ts new file mode 100644 index 0000000000..26a0d4c9c3 --- /dev/null +++ b/ui/src/app/data/interfaces/plugins.ts @@ -0,0 +1,3 @@ +export interface Plugin { + landmarks: string; +} diff --git a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ExceptionResponseDto.java b/ui/src/app/data/interfaces/pose-subjects.ts similarity index 75% rename from java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ExceptionResponseDto.java rename to ui/src/app/data/interfaces/pose-subjects.ts index fbf5c9a407..381d703144 100644 --- a/java/api/src/main/java/com/exadel/frs/core/trainservice/dto/ExceptionResponseDto.java +++ b/ui/src/app/data/interfaces/pose-subjects.ts @@ -14,15 +14,9 @@ * permissions and limitations under the License. */ -package com.exadel.frs.core.trainservice.dto; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class ExceptionResponseDto { - - private String message; - private Integer code; +/* eslint-disable @typescript-eslint/naming-convention */ +export interface PoseSubject { + pitch: number; + roll: number; + yaw: number; } diff --git a/ui/src/app/data/interfaces/request-result-recognition.ts b/ui/src/app/data/interfaces/request-result-recognition.ts index c17e14aef7..d4060cf1a8 100644 --- a/ui/src/app/data/interfaces/request-result-recognition.ts +++ b/ui/src/app/data/interfaces/request-result-recognition.ts @@ -17,11 +17,13 @@ import { BoxSize } from './box-size'; import { BoxSubjects } from './box-subjects'; import { PersonAge } from './person-age'; import { PersonGender } from './person-gender'; +import { PoseSubject } from './pose-subjects'; export interface RequestResultRecognition { age: PersonAge; gender: PersonGender; box: BoxSize; + pose: PoseSubject; subjects: BoxSubjects[]; landmarks: [number[]]; } diff --git a/ui/src/app/data/interfaces/size.interface.ts b/ui/src/app/data/interfaces/size.interface.ts new file mode 100644 index 0000000000..4124fbd869 --- /dev/null +++ b/ui/src/app/data/interfaces/size.interface.ts @@ -0,0 +1,4 @@ +export interface MaxImageSize { + clientMaxFileSize: number; + clientMaxBodySize: number; +} diff --git a/ui/src/app/data/interfaces/source-image-face.ts b/ui/src/app/data/interfaces/source-image-face.ts index a301885780..a7131e494a 100644 --- a/ui/src/app/data/interfaces/source-image-face.ts +++ b/ui/src/app/data/interfaces/source-image-face.ts @@ -16,10 +16,12 @@ import { BoxSize } from './box-size'; import { PersonAge } from './person-age'; import { PersonGender } from './person-gender'; +import { PoseSubject } from './pose-subjects'; export interface SourceImageFace { age: PersonAge; gender: PersonGender; box: BoxSize; + pose: PoseSubject; landmarks: [number[]]; } diff --git a/ui/src/app/data/interfaces/statistics.ts b/ui/src/app/data/interfaces/statistics.ts new file mode 100644 index 0000000000..bebac94edd --- /dev/null +++ b/ui/src/app/data/interfaces/statistics.ts @@ -0,0 +1,4 @@ +export interface Statistics { + requestCount: number; + createdDate: string; +} diff --git a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FaceSimilarityDto.java b/ui/src/app/data/interfaces/user-data.ts similarity index 72% rename from java/common/src/main/java/com/exadel/frs/commonservice/dto/FaceSimilarityDto.java rename to ui/src/app/data/interfaces/user-data.ts index fe96ec6d3d..090639ce73 100644 --- a/java/common/src/main/java/com/exadel/frs/commonservice/dto/FaceSimilarityDto.java +++ b/ui/src/app/data/interfaces/user-data.ts @@ -1,29 +1,22 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.commonservice.dto; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Value; - -@Value -public class FaceSimilarityDto { - - @JsonProperty("face_name") - String faceName; - - float similarity; -} +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +export interface UserData { + role: string; + userId: string; + email: string; + fullName: string; +} diff --git a/ui/src/app/features/alert/alert.component.scss b/ui/src/app/features/alert/alert.component.scss index 0a1bfa9abd..1b3588a94e 100644 --- a/ui/src/app/features/alert/alert.component.scss +++ b/ui/src/app/features/alert/alert.component.scss @@ -16,19 +16,20 @@ @import 'colors.scss'; -.mat-dialog-title, .mat-dialog-content { +.mat-dialog-title, +.mat-dialog-content { text-align: center; } .mat-dialog-actions { display: flex; - flex-direction: column + flex-direction: column; } .alert { &-info { background-color: $green; - color: $white + color: $white; } &-warning { @@ -37,11 +38,10 @@ &-error { background-color: $red; - color: $white + color: $white; } } button { color: $white; } - diff --git a/ui/src/app/features/app-change-photo/app-change-photo.component.html b/ui/src/app/features/app-change-photo/app-change-photo.component.html index 47d14c7442..6fbb551b9f 100644 --- a/ui/src/app/features/app-change-photo/app-change-photo.component.html +++ b/ui/src/app/features/app-change-photo/app-change-photo.component.html @@ -14,22 +14,43 @@ ~ permissions and limitations under the License. -->
- +
+ - + +
+
+ - + +
diff --git a/ui/src/app/features/app-change-photo/app-change-photo.component.scss b/ui/src/app/features/app-change-photo/app-change-photo.component.scss index 0bb019e06b..8a282ab61c 100644 --- a/ui/src/app/features/app-change-photo/app-change-photo.component.scss +++ b/ui/src/app/features/app-change-photo/app-change-photo.component.scss @@ -13,8 +13,8 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -@import "mixins.scss"; -@import "colors.scss"; +@import 'mixins.scss'; +@import 'colors.scss'; .change-photo { position: absolute; @@ -43,7 +43,12 @@ &.landmarks-active { svg path { fill: $special-dark-gray !important; - } + }; + background-color: $bg-light-gray; + } + + &.pose-active { + background-color: $bg-light-gray; } &:last-child { diff --git a/ui/src/app/features/app-change-photo/app-change-photo.component.ts b/ui/src/app/features/app-change-photo/app-change-photo.component.ts index 65d4e08b6c..e876be83d1 100644 --- a/ui/src/app/features/app-change-photo/app-change-photo.component.ts +++ b/ui/src/app/features/app-change-photo/app-change-photo.component.ts @@ -24,12 +24,15 @@ import { ChangeDetectionStrategy, Component, Output, EventEmitter, ViewChild, El export class AppChangePhotoComponent { @Input() disabledButtons: boolean; @Input() disabledLandmarksButton: boolean; + @Input() disabledPoseButton: boolean; @Output() changePhoto = new EventEmitter(); @Output() resetPhoto = new EventEmitter(); @Output() addLandmark = new EventEmitter(); + @Output() addPose = new EventEmitter(); @ViewChild('uploadFile') fileDropEl: ElementRef; showLandmarks = false; + showPose = false; } diff --git a/ui/src/app/features/app-change-photo/app-change-photo.module.ts b/ui/src/app/features/app-change-photo/app-change-photo.module.ts index 62a44f76b9..469d981066 100644 --- a/ui/src/app/features/app-change-photo/app-change-photo.module.ts +++ b/ui/src/app/features/app-change-photo/app-change-photo.module.ts @@ -16,12 +16,14 @@ import { NgModule } from '@angular/core'; import { MatIconModule } from '@angular/material/icon'; import { MatButtonModule } from '@angular/material/button'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { TranslateModule } from '@ngx-translate/core'; import { AppChangePhotoComponent } from './app-change-photo.component'; @NgModule({ declarations: [AppChangePhotoComponent], exports: [AppChangePhotoComponent], - imports: [MatIconModule, MatButtonModule], + imports: [MatIconModule, MatButtonModule, MatTooltipModule, TranslateModule], }) export class AppChangePhotoModule {} diff --git a/ui/src/app/features/app-search-table/app-search-table.component.html b/ui/src/app/features/app-search-table/app-search-table.component.html index 7be39b3dfc..ae4f5b7a28 100644 --- a/ui/src/app/features/app-search-table/app-search-table.component.html +++ b/ui/src/app/features/app-search-table/app-search-table.component.html @@ -20,13 +20,17 @@
- +
+ + +
-
diff --git a/ui/src/app/features/app-search-table/app-search-table.component.scss b/ui/src/app/features/app-search-table/app-search-table.component.scss index d3ac9a0dd2..ce5339864f 100644 --- a/ui/src/app/features/app-search-table/app-search-table.component.scss +++ b/ui/src/app/features/app-search-table/app-search-table.component.scss @@ -14,15 +14,15 @@ * permissions and limitations under the License. */ -@import "colors.scss"; -@import "mixins.scss"; -@import "media.scss"; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'media.scss'; .app-search { - margin-bottom: 20px; + padding-bottom: 20px; display: flex; align-items: center; - justify-content: space-between; + border-bottom: 1px solid $lighter-gray; @include mobile-landscape; @include mobile-portrait; @@ -31,6 +31,9 @@ } &--title { + font-size: $text-lg; + line-height: 53px; + color: $dark-blue; margin: 0 16px 0 0; @include mobile { @@ -39,8 +42,10 @@ } &--form { - flex-grow: 1; position: relative; + width: 30%; + max-width: 483px; + margin-right: auto; @include mobile-landscape; @include mobile-portrait; @@ -54,19 +59,21 @@ &_input { padding: 8px 25px 8px 8px; width: 100%; - border: 1px solid $super-light-gray; - border-radius: 4px; - font-size: 14px; + border: none; + border-bottom: 1px solid $light-gray; + background: none; + font-size: $text-sm; box-sizing: border-box; - outline-color: $primary; + outline: none; &::placeholder { color: $gray; + font-size: $text-sm; } } .mat-icon { - @include sm-icon-size(); + @include sm-icon-size(20px); position: absolute; top: 50%; right: 8px; @@ -78,22 +85,24 @@ margin-left: 8px; padding: 0; - @include mobile { - margin: 0; + &_user_list { + cursor: pointer; + display: flex; + align-items: center; + border: none; + background: none; + .mat-icon { + @include sm-icon-size(23px); + } } button { - padding: 10px 8px; - line-height: 10px; - } + @include action-btns(193px, $white, $light-blue); + border: none; - .mat-icon { - @include sm-icon-size(); - margin-right: 5px; - } - - &_text { - color: $primary; + span { + margin-left: 5px; + } } } } diff --git a/ui/src/app/features/app-search-table/app-search-table.component.ts b/ui/src/app/features/app-search-table/app-search-table.component.ts index dd8c406abc..bb14f335f6 100644 --- a/ui/src/app/features/app-search-table/app-search-table.component.ts +++ b/ui/src/app/features/app-search-table/app-search-table.component.ts @@ -28,7 +28,9 @@ export class AppSearchTableComponent { @Input() buttonText: string; @Input() requiredRole: string; @Input() currentRole: string; + @Input() hideContent: boolean; + @Output() manageUsersView = new EventEmitter(); @Output() inputSearch: EventEmitter = new EventEmitter(); @Output() modalWindow: EventEmitter = new EventEmitter(); @@ -40,4 +42,8 @@ export class AppSearchTableComponent { onButtonChange(event: MouseEvent): void { this.modalWindow.emit(event); } + + onOpenUserList() { + this.manageUsersView.emit(); + } } diff --git a/ui/src/app/features/app-search-table/app-search-table.module.ts b/ui/src/app/features/app-search-table/app-search-table.module.ts index 321d20ceeb..1069b22e08 100644 --- a/ui/src/app/features/app-search-table/app-search-table.module.ts +++ b/ui/src/app/features/app-search-table/app-search-table.module.ts @@ -23,10 +23,12 @@ import { MatInputModule } from '@angular/material/input'; import { AppSearchTableComponent } from './app-search-table.component'; import { MatButtonModule } from '@angular/material/button'; +import { TruncateModule } from 'src/app/ui/truncate-pipe/truncate.module'; +import { MatTooltipModule } from '@angular/material/tooltip'; @NgModule({ declarations: [AppSearchTableComponent], exports: [AppSearchTableComponent], - imports: [CommonModule, TranslateModule, MatIconModule, FormsModule, MatInputModule, MatButtonModule], + imports: [CommonModule, TranslateModule, MatIconModule, FormsModule, MatInputModule, MatButtonModule, TruncateModule, MatTooltipModule], }) export class AppSearchTableModule {} diff --git a/ui/src/app/features/app-user-list/application-user-list-facade.ts b/ui/src/app/features/app-user-list/application-user-list-facade.ts deleted file mode 100644 index 5e1e1c3833..0000000000 --- a/ui/src/app/features/app-user-list/application-user-list-facade.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -import { Injectable } from '@angular/core'; -import { Store } from '@ngrx/store'; -import { Observable, Subscription } from 'rxjs'; -import { AppUser } from 'src/app/data/interfaces/app-user'; -import { IFacade } from 'src/app/data/interfaces/IFacade'; -import { AppState } from 'src/app/store'; -import { deleteUserFromApplication, inviteAppUser, loadAppUserEntityAction, updateAppUserRole } from 'src/app/store/app-user/action'; -import { - selectAppUsers, - selectAvailableEmails, - selectAvailableRoles, - selectIsApplicationLoading, - selectUserRole, -} from 'src/app/store/app-user/selectors'; -import { selectCurrentApp, selectUserRollForSelectedApp } from 'src/app/store/application/selectors'; -import { loadRolesEntity } from 'src/app/store/role/action'; -import { selectUserId } from 'src/app/store/userInfo/selectors'; - -import { loadUsersEntity } from '../../store/user/action'; -import { selectCurrentUserRole } from '../../store/user/selectors'; -import { Role } from 'src/app/data/enums/role.enum'; - -@Injectable() -export class ApplicationUserListFacade implements IFacade { - isLoading$: Observable; - appUsers$: Observable; - availableRoles$: Observable; - availableEmails$: Observable; - userRole$: Observable; - currentUserId$: Observable; - selectedApplicationName: string; - - private selectedApplicationId: string; - private sub: Subscription; - userGlobalRole$: Observable; - applicationRole$: Observable; - - constructor(private store: Store) { - this.appUsers$ = this.store.select(selectAppUsers); - this.availableEmails$ = this.store.select(selectAvailableEmails); - this.userGlobalRole$ = this.store.select(selectCurrentUserRole); - this.applicationRole$ = this.store.select(selectUserRollForSelectedApp); - this.userRole$ = this.store.select(selectUserRole); - this.currentUserId$ = this.store.select(selectUserId); - this.availableRoles$ = this.store.select(selectAvailableRoles); - this.isLoading$ = this.store.select(selectIsApplicationLoading); - } - - initSubscriptions(): void { - this.sub = this.store.select(selectCurrentApp).subscribe(app => { - if (app) { - this.selectedApplicationId = app.id; - this.selectedApplicationName = app.name; - this.loadData(); - } - }); - } - - loadData(): void { - this.store.dispatch( - loadAppUserEntityAction({ - applicationId: this.selectedApplicationId, - }) - ); - this.store.dispatch(loadRolesEntity()); - this.store.dispatch(loadUsersEntity()); - } - - updateUserRole(id: string, role: Role): void { - this.store.dispatch( - updateAppUserRole({ - applicationId: this.selectedApplicationId, - user: { - id, - role, - }, - }) - ); - } - - inviteUser(email: string, role: Role): void { - this.store.dispatch( - inviteAppUser({ - applicationId: this.selectedApplicationId, - userEmail: email, - role, - }) - ); - } - - delete(userId: string) { - this.store.dispatch( - deleteUserFromApplication({ - applicationId: this.selectedApplicationId, - userId, - }) - ); - } - - unsubscribe(): void { - this.sub.unsubscribe(); - } -} diff --git a/ui/src/app/features/app-user-list/application-user-list.component.html b/ui/src/app/features/app-user-list/application-user-list.component.html deleted file mode 100644 index efe62d760d..0000000000 --- a/ui/src/app/features/app-user-list/application-user-list.component.html +++ /dev/null @@ -1,42 +0,0 @@ - - -
- - - - - -
diff --git a/ui/src/app/features/app-user-list/application-user-list.component.spec.ts b/ui/src/app/features/app-user-list/application-user-list.component.spec.ts deleted file mode 100644 index 181bb34a4e..0000000000 --- a/ui/src/app/features/app-user-list/application-user-list.component.spec.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; -import { FormsModule } from '@angular/forms'; -import { MatDialog } from '@angular/material/dialog'; -import { MatInputModule } from '@angular/material/input'; -import { NoopAnimationsModule } from '@angular/platform-browser/animations'; -import { TranslatePipe, TranslateService } from '@ngx-translate/core'; -import { of, Subscription } from 'rxjs'; -import { SnackBarModule } from 'src/app/features/snackbar/snackbar.module'; - -import { TablePipeModule } from '../../ui/search-pipe/table-filter.module'; -import { SpinnerModule } from '../spinner/spinner.module'; -import { UserTableModule } from '../user-table/user-table.module'; -import { ApplicationUserListFacade } from './application-user-list-facade'; -import { ApplicationUserListComponent } from './application-user-list.component'; - -describe('ApplicationUserListComponent', () => { - let component: ApplicationUserListComponent; - let fixture: ComponentFixture; - - beforeEach( - waitForAsync(() => { - TestBed.configureTestingModule({ - declarations: [ApplicationUserListComponent, TranslatePipe], - imports: [SpinnerModule, UserTableModule, NoopAnimationsModule, FormsModule, TablePipeModule, MatInputModule, SnackBarModule], - providers: [ - { - provide: MatDialog, - useValue: {}, - }, - { - provide: ApplicationUserListFacade, - useValue: { - initSubscriptions: () => of([{}]), - appUsers$: of([ - { - id: 0, - name: 'name', - owner: { - firstname: 'firstname', - }, - }, - ]), - isLoading$: of([{}]), - availableRoles$: of([{}]), - unsubscribe: () => {}, - }, - }, - { provide: TranslateService, useValue: {} }, - ], - }).compileComponents(); - }) - ); - - beforeEach(() => { - fixture = TestBed.createComponent(ApplicationUserListComponent); - component = fixture.componentInstance; - component.availableRolesSubscription = new Subscription(); - }); - - it('should create', () => { - expect(component).toBeDefined(); - }); -}); diff --git a/ui/src/app/features/app-user-list/application-user-list.component.ts b/ui/src/app/features/app-user-list/application-user-list.component.ts deleted file mode 100644 index 9664cbb77b..0000000000 --- a/ui/src/app/features/app-user-list/application-user-list.component.ts +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -import { ChangeDetectionStrategy, Component, OnDestroy, OnInit } from '@angular/core'; -import { MatDialog } from '@angular/material/dialog'; -import { TranslateService } from '@ngx-translate/core'; -import { Observable, Subscription } from 'rxjs'; -import { filter, first, map, takeWhile } from 'rxjs/operators'; -import { Role } from 'src/app/data/enums/role.enum'; -import { AppUser } from 'src/app/data/interfaces/app-user'; - -import { UserDeletion } from '../../data/interfaces/user-deletion'; -import { DeleteDialogComponent } from '../delete-dialog/delete-dialog.component'; -import { InviteDialogComponent } from '../invite-dialog/invite-dialog.component'; -import { SnackBarService } from '../snackbar/snackbar.service'; -import { ITableConfig } from '../table/table.component'; -import { ApplicationUserListFacade } from './application-user-list-facade'; - -@Component({ - selector: 'app-application-user-list', - templateUrl: './application-user-list.component.html', - styleUrls: ['./application-user-list.component.scss'], - changeDetection: ChangeDetectionStrategy.OnPush, -}) -export class ApplicationUserListComponent implements OnInit, OnDestroy { - tableConfig$: Observable; - isLoading$: Observable; - userRole$: Observable; - availableRoles$: Observable; - availableEmails$: Observable; - message: string; - search = ''; - availableRoles: string[]; - currentUserId$: Observable; - roleEnum = Role; - availableRolesSubscription: Subscription; - - constructor( - private appUserListFacade: ApplicationUserListFacade, - private dialog: MatDialog, - private snackBarService: SnackBarService, - private translate: TranslateService - ) { - appUserListFacade.initSubscriptions(); - } - - ngOnInit() { - this.isLoading$ = this.appUserListFacade.isLoading$; - this.userRole$ = this.appUserListFacade.userRole$; - this.availableEmails$ = this.appUserListFacade.availableEmails$; - this.currentUserId$ = this.appUserListFacade.currentUserId$; - - this.tableConfig$ = this.appUserListFacade.appUsers$.pipe( - map((users: AppUser[]) => ({ - columns: [ - { title: 'user', property: 'username' }, - { title: 'role', property: 'role' }, - { title: 'delete', property: 'delete' }, - ], - data: users, - })) - ); - this.message = this.translate.instant('app_users.add_users_info'); - this.availableRoles$ = this.appUserListFacade.availableRoles$; - this.availableRolesSubscription = this.appUserListFacade.availableRoles$.subscribe(value => (this.availableRoles = value)); - } - - onChange(user: AppUser): void { - this.appUserListFacade.updateUserRole(user.id, user.role); - } - - onSearch(value: string) { - this.search = value; - } - - onDelete(deletion: UserDeletion): void { - const dialog = this.dialog.open(DeleteDialogComponent, { - panelClass: 'custom-mat-dialog', - data: { - entityType: this.translate.instant('users.user'), - entityName: `${deletion.userToDelete.firstName} ${deletion.userToDelete.lastName}`, - applicationName: this.appUserListFacade.selectedApplicationName, - }, - }); - - dialog - .afterClosed() - .pipe( - first(), - filter(result => result) - ) - .subscribe(() => this.appUserListFacade.delete(deletion.userToDelete.userId)); - } - - ngOnDestroy(): void { - this.appUserListFacade.unsubscribe(); - this.availableRolesSubscription.unsubscribe(); - } - - onInviteUser(): void { - const dialog = this.dialog.open(InviteDialogComponent, { - panelClass: 'custom-mat-dialog', - data: { - availableRoles: this.availableRoles, - options$: this.availableEmails$, - }, - }); - - dialog - .afterClosed() - .pipe(takeWhile(({ userEmail, role }) => userEmail && role)) - .subscribe(({ userEmail, role }) => { - this.appUserListFacade.inviteUser(userEmail, role); - }); - } -} diff --git a/ui/src/app/features/application-collection-container/application-collection-container.component.html b/ui/src/app/features/application-collection-container/application-collection-container.component.html new file mode 100644 index 0000000000..60c9f2d84a --- /dev/null +++ b/ui/src/app/features/application-collection-container/application-collection-container.component.html @@ -0,0 +1,34 @@ + + +
+ + + + + +
+ +

+ {{ 'users.search.no_results' | translate }} +

+
+ +
diff --git a/ui/src/app/pages/manage-collection/manage-collection.component.scss b/ui/src/app/features/application-collection-container/application-collection-container.component.scss similarity index 59% rename from ui/src/app/pages/manage-collection/manage-collection.component.scss rename to ui/src/app/features/application-collection-container/application-collection-container.component.scss index 16aee37aef..50d94d0446 100644 --- a/ui/src/app/pages/manage-collection/manage-collection.component.scss +++ b/ui/src/app/features/application-collection-container/application-collection-container.component.scss @@ -14,24 +14,27 @@ * permissions and limitations under the License. */ -.collection { - &--grid { - display: grid; - grid-template-columns: 220px 1fr; - height: calc(100vh - 254px); - grid-gap: 17px; - grid-template-areas: 'subjects-list profile'; +@import 'mixins.scss'; +@import 'media.scss'; - &_subjects-list { - grid-area: subjects-list; - padding: 24px 8px !important; - height: 100%; - } +.application-container { + display: flex; + flex-wrap: wrap; - &_profile { - grid-area: profile; - height: 100%; - min-width: 100%; - } + @include mobile-landscape; + @include mobile-portrait; + @include mobile { + justify-content: center; + } +} +.no-data-message { + display: flex; + align-items: center; + p { + @include no-data-msg; + } + .mat-icon { + @include sm-icon-size(14px); + margin-right: 10px; } } diff --git a/ui/src/app/features/application-collection-container/application-collection-container.component.ts b/ui/src/app/features/application-collection-container/application-collection-container.component.ts new file mode 100644 index 0000000000..39dbd848a9 --- /dev/null +++ b/ui/src/app/features/application-collection-container/application-collection-container.component.ts @@ -0,0 +1,32 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { ChangeDetectionStrategy } from '@angular/core'; +import { Component, EventEmitter, Input, Output } from '@angular/core'; +import { Application } from 'src/app/data/interfaces/application'; + +@Component({ + selector: 'application-collection-container', + templateUrl: './application-collection-container.component.html', + styleUrls: ['./application-collection-container.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ApplicationCollectionContainerComponent { + @Output() selectApp = new EventEmitter(); + @Input() applicationCollection: Application[]; + @Input() isLoading: boolean; + @Input() totalApplications: number; +} diff --git a/ui/src/app/features/application-collection-container/application-collection-container.module.ts b/ui/src/app/features/application-collection-container/application-collection-container.module.ts new file mode 100644 index 0000000000..3169972d11 --- /dev/null +++ b/ui/src/app/features/application-collection-container/application-collection-container.module.ts @@ -0,0 +1,32 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { CommonModule } from '@angular/common'; +import { NgModule } from '@angular/core'; +import { MatIconModule } from '@angular/material/icon'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { TranslateModule } from '@ngx-translate/core'; +import { TablePipeModule } from 'src/app/ui/search-pipe/table-filter.module'; +import { TruncateModule } from 'src/app/ui/truncate-pipe/truncate.module'; +import { ApplicationCollectionContainerComponent } from './application-collection-container.component'; +import { ApplicationCollectionComponent } from './application-collection/application-collection.component'; + +@NgModule({ + declarations: [ApplicationCollectionContainerComponent, ApplicationCollectionComponent], + imports: [CommonModule, TranslateModule, TablePipeModule, MatIconModule,MatTooltipModule, TruncateModule], + exports: [ApplicationCollectionContainerComponent, ApplicationCollectionComponent], +}) +export class ApplicationCollectionContainerModule {} diff --git a/ui/src/app/features/application-collection-container/application-collection/application-collection.component.html b/ui/src/app/features/application-collection-container/application-collection/application-collection.component.html new file mode 100644 index 0000000000..a75005c2f7 --- /dev/null +++ b/ui/src/app/features/application-collection-container/application-collection/application-collection.component.html @@ -0,0 +1,40 @@ + + + +
+
+ + {{ element.name | truncate: 15 }} + +
+
+ + {{ fullName | truncate: 30 }} + +
+
+ diff --git a/ui/src/app/features/application-collection-container/application-collection/application-collection.component.scss b/ui/src/app/features/application-collection-container/application-collection/application-collection.component.scss new file mode 100644 index 0000000000..20a38dd870 --- /dev/null +++ b/ui/src/app/features/application-collection-container/application-collection/application-collection.component.scss @@ -0,0 +1,70 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'colors.scss'; +@import 'sizes.scss'; + +.app { + cursor: pointer; + display: flex; + justify-content: center; + align-items: center; + flex-direction: column; + border-radius: 30px; + background: $bg-light-gray; + width: 275px; + height: 229px; + margin: 40px 18px 0; + + &__container { + display: flex; + justify-content: center; + margin-bottom: 20px; + width: 100%; + + &--title { + color: $dark-blue; + font-size: $text-lg; + word-break: break-all; + line-height: 2rem; + width: 90%; + text-align: center; + } + &--owner { + color: $pale-blue; + font-size: $text-sm; + max-width: 200px; + word-wrap: break-word; + text-align: center; + } + } + + &:hover { + background: linear-gradient($primary 0%, $primary-lighter 100%); + box-shadow: 0px 8px 14px $light-gray; + border: 1px solid $light-blue; + + .app__container { + &--title { + color: $white; + } + &--owner { + max-height: 40px; + color: $white; + } + } + } +} diff --git a/ui/src/app/features/application-collection-container/application-collection/application-collection.component.ts b/ui/src/app/features/application-collection-container/application-collection/application-collection.component.ts new file mode 100644 index 0000000000..6900bd3565 --- /dev/null +++ b/ui/src/app/features/application-collection-container/application-collection/application-collection.component.ts @@ -0,0 +1,35 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { ChangeDetectionStrategy, Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; +import { Application } from 'src/app/data/interfaces/application'; + +@Component({ + selector: 'application-collection', + templateUrl: './application-collection.component.html', + styleUrls: ['./application-collection.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ApplicationCollectionComponent implements OnInit { + @Output() selectApp = new EventEmitter(); + @Input() element: Application; + + fullName: string; + + ngOnInit(): void { + this.fullName = this.element.owner.firstName + ' ' + this.element.owner.lastName; + } +} diff --git a/ui/src/app/features/application-header/application-header.container.component.ts b/ui/src/app/features/application-header/application-header.container.component.ts index 82409d5ec8..8368d0a0f2 100644 --- a/ui/src/app/features/application-header/application-header.container.component.ts +++ b/ui/src/app/features/application-header/application-header.container.component.ts @@ -18,11 +18,8 @@ import { Component, OnInit } from '@angular/core'; import { MatDialog } from '@angular/material/dialog'; import { TranslateService } from '@ngx-translate/core'; import { Observable } from 'rxjs'; -import { filter, first } from 'rxjs/operators'; import { Application } from '../../data/interfaces/application'; -import { DeleteDialogComponent } from '../delete-dialog/delete-dialog.component'; -import { EditDialogComponent } from '../edit-dialog/edit-dialog.component'; import { ApplicationHeaderFacade } from './application-header.facade'; @Component({ @@ -32,8 +29,6 @@ import { ApplicationHeaderFacade } from './application-header.facade'; [app]="app$ | async" [isLoading]="isLoading$ | async" [userRole]="userRole$ | async" - (rename)="rename($event)" - (delete)="delete($event)" > `, }) @@ -51,40 +46,4 @@ export class ApplicationHeaderContainerComponent implements OnInit { this.userRole$ = this.applicationHeaderFacade.userRole$; this.isLoading$ = this.applicationHeaderFacade.isLoadingAppList$; } - - rename(name: string): void { - const dialog = this.dialog.open(EditDialogComponent, { - panelClass: 'custom-mat-dialog', - data: { - entityType: this.translate.instant('applications.header.title'), - entityName: name, - }, - }); - - dialog - .afterClosed() - .pipe( - first(), - filter(result => result) - ) - .subscribe(result => this.applicationHeaderFacade.rename(result)); - } - - delete(name: string) { - const dialog = this.dialog.open(DeleteDialogComponent, { - panelClass: 'custom-mat-dialog', - data: { - entityType: this.translate.instant('applications.header.title'), - entityName: name, - }, - }); - - dialog - .afterClosed() - .pipe( - first(), - filter(result => result) - ) - .subscribe(() => this.applicationHeaderFacade.delete()); - } } diff --git a/ui/src/app/features/application-header/application-header/application-header.component.html b/ui/src/app/features/application-header/application-header/application-header.component.html index b0a150d9d4..5e9d1d6e90 100644 --- a/ui/src/app/features/application-header/application-header/application-header.component.html +++ b/ui/src/app/features/application-header/application-header/application-header.component.html @@ -23,10 +23,10 @@

{{ app.name }}

diff --git a/ui/src/app/features/application-header/application-header/application-header.component.scss b/ui/src/app/features/application-header/application-header/application-header.component.scss index c899e61203..6194595e69 100644 --- a/ui/src/app/features/application-header/application-header/application-header.component.scss +++ b/ui/src/app/features/application-header/application-header/application-header.component.scss @@ -13,7 +13,7 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -@import "mixins.scss"; +@import 'mixins.scss'; .app-header { margin-bottom: 24px; diff --git a/ui/src/app/features/application-header/application-header/application-header.component.ts b/ui/src/app/features/application-header/application-header/application-header.component.ts index ae261379c4..d1f7cdb283 100644 --- a/ui/src/app/features/application-header/application-header/application-header.component.ts +++ b/ui/src/app/features/application-header/application-header/application-header.component.ts @@ -28,19 +28,9 @@ export class ApplicationHeaderComponent { @Input() app: Application; @Input() isLoading: boolean; @Input() userRole: string; - @Output() rename = new EventEmitter(); - @Output() delete = new EventEmitter(); maxHeaderLinkLength = 25; - onRename(name: string): void { - this.rename.emit(name); - } - - onDelete(name: string): void { - this.delete.emit(name); - } - // Users who can edit application checkUserRole(role: string): boolean { return Role.Administrator === role || Role.Owner === role; diff --git a/ui/src/app/features/application-list/application-list-container.component.ts b/ui/src/app/features/application-list/application-list-container.component.ts index 9f6f75ae08..985362c15a 100644 --- a/ui/src/app/features/application-list/application-list-container.component.ts +++ b/ui/src/app/features/application-list/application-list-container.component.ts @@ -13,16 +13,17 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, OnDestroy, OnInit } from '@angular/core'; import { MatDialog } from '@angular/material/dialog'; import { Router } from '@angular/router'; import { TranslateService } from '@ngx-translate/core'; -import { Observable } from 'rxjs'; -import { map } from 'rxjs/operators'; +import { Observable, Subscription } from 'rxjs'; +import { AppUser } from 'src/app/data/interfaces/app-user'; +import { Application } from 'src/app/data/interfaces/application'; import { CreateDialogComponent } from 'src/app/features/create-dialog/create-dialog.component'; -import { ITableConfig } from 'src/app/features/table/table.component'; import { Routes } from '../../data/enums/routers-url.enum'; +import { ManageUsersDialog } from '../manage-users-dialog/manage-users.component'; import { ApplicationListFacade } from './application-list-facade'; @Component({ @@ -31,18 +32,23 @@ import { ApplicationListFacade } from './application-list-facade'; `, changeDetection: ChangeDetectionStrategy.OnPush, }) -export class ApplicationListContainerComponent implements OnInit { +export class ApplicationListContainerComponent implements OnInit, OnDestroy { isLoading$: Observable; userRole$: Observable; - tableConfig$: Observable; + users$: Observable; + currentUserId$: Observable; + + applications: Application[]; + subs: Subscription; constructor( private applicationFacade: ApplicationListFacade, @@ -56,13 +62,9 @@ export class ApplicationListContainerComponent implements OnInit { ngOnInit() { this.isLoading$ = this.applicationFacade.isLoading$; this.userRole$ = this.applicationFacade.userRole$; - - this.tableConfig$ = this.applicationFacade.applications$.pipe( - map(apps => ({ - columns: [{ title: 'name', property: 'name' }], - data: apps.map(app => ({ id: app.id, name: app.name, owner: `${app.owner.firstName} ${app.owner.lastName}` })), - })) - ); + this.users$ = this.applicationFacade.appUsers$; + this.currentUserId$ = this.applicationFacade.currentUserId$; + this.subs = this.applicationFacade.applications$.subscribe(applications => (this.applications = applications)); } onClick(application): void { @@ -74,11 +76,15 @@ export class ApplicationListContainerComponent implements OnInit { } onCreateNewApp(): void { + const applicationNames = this.applications.map(app => app.name); + const dialog = this.dialog.open(CreateDialogComponent, { panelClass: 'custom-mat-dialog', data: { entityType: this.translate.instant('applications.header.title'), placeholder: this.translate.instant('applications.name'), + errorMsg: this.translate.instant('applications.error_msg'), + nameList: applicationNames, name: '', }, }); @@ -90,4 +96,35 @@ export class ApplicationListContainerComponent implements OnInit { } }); } + + onManageUsers() { + let userCollection; + let userId; + let userRole; + + const userSubs = this.users$.subscribe(res => (userCollection = res)); + + const userIdSubs = this.currentUserId$.subscribe(res => (userId = res)); + + const userRoleSubs = this.userRole$.subscribe(res => (userRole = res)); + + const dialog = this.dialog.open(ManageUsersDialog, { + data: { + collection: this.users$, + currentUserId: userId, + currentUserRole: userRole, + applications: this.applications, + }, + }); + + const dialogSubs = dialog.afterClosed().subscribe(() => { + userSubs.unsubscribe(); + userIdSubs.unsubscribe(); + userRoleSubs.unsubscribe(); + }); + } + + ngOnDestroy(): void { + this.subs.unsubscribe(); + } } diff --git a/ui/src/app/features/application-list/application-list-facade.ts b/ui/src/app/features/application-list/application-list-facade.ts index 9f702e0b09..878876e1d2 100644 --- a/ui/src/app/features/application-list/application-list-facade.ts +++ b/ui/src/app/features/application-list/application-list-facade.ts @@ -16,27 +16,47 @@ import { Injectable } from '@angular/core'; import { Store } from '@ngrx/store'; import { Observable } from 'rxjs'; +import { Role } from 'src/app/data/enums/role.enum'; +import { AppUser } from 'src/app/data/interfaces/app-user'; import { Application } from 'src/app/data/interfaces/application'; import { IFacade } from 'src/app/data/interfaces/IFacade'; +import { UserDeletion } from 'src/app/data/interfaces/user-deletion'; import { AppState } from 'src/app/store'; import { createApplication, loadApplications } from 'src/app/store/application/action'; import { selectApplications, selectIsPendingApplicationList } from 'src/app/store/application/selectors'; -import { selectCurrentUserRole } from 'src/app/store/user/selectors'; +import { loadRolesEntity } from 'src/app/store/role/action'; +import { deleteUser, loadUsersEntity, updateUserRoleWithRefresh } from 'src/app/store/user/action'; +import { selectCurrentUserRole, selectUsers } from 'src/app/store/user/selectors'; +import { selectUserId } from 'src/app/store/userInfo/selectors'; @Injectable() export class ApplicationListFacade implements IFacade { applications$: Observable; isLoading$: Observable; userRole$: Observable; + appUsers$: Observable; + currentUserId$: Observable; constructor(private store: Store) { this.applications$ = this.store.select(selectApplications); this.userRole$ = this.store.select(selectCurrentUserRole); this.isLoading$ = this.store.select(selectIsPendingApplicationList); + this.appUsers$ = this.store.select(selectUsers); + this.currentUserId$ = this.store.select(selectUserId); } initSubscriptions(): void { this.loadApplications(); + this.loadUsers(); + this.loadAvailableRoles(); + } + + loadUsers(): void { + this.store.dispatch(loadUsersEntity()); + } + + loadAvailableRoles(): void { + this.store.dispatch(loadRolesEntity()); } loadApplications(): void { @@ -46,4 +66,25 @@ export class ApplicationListFacade implements IFacade { createApplication(name: string): void { this.store.dispatch(createApplication({ name })); } + + updateUserRole(id: string, role: Role): void { + this.store.dispatch( + updateUserRoleWithRefresh({ + user: { + id, + role, + }, + }) + ); + } + + deleteUser(deletion: UserDeletion, newOwner: string): void { + this.store.dispatch( + deleteUser({ + userId: deletion.userToDelete.userId, + deleterUserId: deletion.deleterUserId, + newOwner, + }) + ); + } } diff --git a/ui/src/app/features/application-list/application-list.module.ts b/ui/src/app/features/application-list/application-list.module.ts index 7cd4562af1..6bdc099f9e 100644 --- a/ui/src/app/features/application-list/application-list.module.ts +++ b/ui/src/app/features/application-list/application-list.module.ts @@ -22,7 +22,6 @@ import { TranslateModule } from '@ngx-translate/core'; import { SnackBarModule } from 'src/app/features/snackbar/snackbar.module'; import { SpinnerModule } from 'src/app/features/spinner/spinner.module'; -import { TableModule } from '../table/table.module'; import { ApplicationListContainerComponent } from './application-list-container.component'; import { ApplicationListFacade } from './application-list-facade'; import { ApplicationListComponent } from './application-list/application-list.component'; @@ -30,6 +29,8 @@ import { FormsModule } from '@angular/forms'; import { TablePipeModule } from '../../ui/search-pipe/table-filter.module'; import { MatInputModule } from '@angular/material/input'; import { AppSearchTableModule } from '../app-search-table/app-search-table.module'; +import { ApplicationCollectionContainerModule } from '../application-collection-container/application-collection-container.module'; +import { ManageUsersModule } from '../manage-users-dialog/manage-users.module'; @NgModule({ declarations: [ApplicationListContainerComponent, ApplicationListComponent], @@ -37,7 +38,6 @@ import { AppSearchTableModule } from '../app-search-table/app-search-table.modul providers: [ApplicationListFacade], imports: [ CommonModule, - TableModule, SpinnerModule, MatButtonModule, SnackBarModule, @@ -48,6 +48,8 @@ import { AppSearchTableModule } from '../app-search-table/app-search-table.modul TablePipeModule, MatInputModule, AppSearchTableModule, + ApplicationCollectionContainerModule, + ManageUsersModule, ], }) export class ApplicationListModule {} diff --git a/ui/src/app/features/application-list/application-list/application-list.component.html b/ui/src/app/features/application-list/application-list/application-list.component.html index 1e8cf2752c..71943783d7 100644 --- a/ui/src/app/features/application-list/application-list/application-list.component.html +++ b/ui/src/app/features/application-list/application-list/application-list.component.html @@ -19,22 +19,24 @@ [requiredRole]="roleEnum.User" (inputSearch)="onSearch($event)" (modalWindow)="createApp.emit()" + (manageUsersView)="manageUsers.emit()" [title]="searchTitle()" buttonText="{{ 'applications.create.add_button' | translate }}" searchPlaceholder="{{ 'applications.search.title' | translate }}" > - - -
- -

{{ 'applications.first_steps_info' | translate }}

+ + +
+

{{ 'applications.no_application_msg' | translate }}

diff --git a/ui/src/app/features/application-list/application-list/application-list.component.scss b/ui/src/app/features/application-list/application-list/application-list.component.scss index a9962d59ae..657e499bf5 100644 --- a/ui/src/app/features/application-list/application-list/application-list.component.scss +++ b/ui/src/app/features/application-list/application-list/application-list.component.scss @@ -14,7 +14,19 @@ * limitations under the License. */ +@import 'sizes.scss'; + app-spinner { position: absolute; left: 40%; } +.no-application-msg { + margin-top: 60px; + width: 70%; + p { + max-width: 656px; + font-weight: 300; + font-size: $text-lg; + line-height: 38px; + } +} diff --git a/ui/src/app/features/application-list/application-list/application-list.component.ts b/ui/src/app/features/application-list/application-list/application-list.component.ts index b26a7f3026..89b823774f 100644 --- a/ui/src/app/features/application-list/application-list/application-list.component.ts +++ b/ui/src/app/features/application-list/application-list/application-list.component.ts @@ -15,8 +15,8 @@ */ import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output } from '@angular/core'; import { Role } from 'src/app/data/enums/role.enum'; -import { ITableConfig } from '../../table/table.component'; import { TranslateService } from '@ngx-translate/core'; +import { Application } from 'src/app/data/interfaces/application'; @Component({ selector: 'app-application-list', @@ -26,11 +26,12 @@ import { TranslateService } from '@ngx-translate/core'; }) export class ApplicationListComponent { @Input() isLoading: boolean; - @Input() tableConfig: ITableConfig; + @Input() applicationCollection: Application[]; @Input() userRole: string; @Output() selectApp = new EventEmitter(); @Output() createApp = new EventEmitter(); + @Output() manageUsers = new EventEmitter(); roleEnum = Role; search = ''; @@ -45,6 +46,6 @@ export class ApplicationListComponent { const titleApp: string = this.translate.instant('applications.title'); const titleCreate: string = this.translate.instant('applications.first_steps_title'); - return this.tableConfig.data.length > 0 ? titleApp : titleCreate; + return this.applicationCollection.length > 0 ? titleApp : titleCreate; } } diff --git a/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.component.spec.ts b/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.component.spec.ts index a28bfb5a91..37b6919200 100644 --- a/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.component.spec.ts +++ b/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.component.spec.ts @@ -14,6 +14,9 @@ * permissions and limitations under the License. */ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; +import { MatDialog } from '@angular/material/dialog'; +import { ActivatedRoute, Router } from '@angular/router'; +import { TranslateService } from '@ngx-translate/core'; import { BreadcrumbsFacade } from '../breadcrumbs/breadcrumbs.facade'; import { BreadcrumbsContainerComponent } from './breadcrumbs.container.component'; @@ -26,7 +29,13 @@ describe('Breadcrumbs.ContainerComponent', () => { waitForAsync(() => { TestBed.configureTestingModule({ declarations: [BreadcrumbsContainerComponent], - providers: [{ provide: BreadcrumbsFacade, useValue: {} }], + providers: [ + { provide: Router, useValue: {} }, + { provide: ActivatedRoute, useValue: {} }, + { provide: MatDialog, useValue: {} }, + { provide: BreadcrumbsFacade, useValue: {} }, + { provide: TranslateService, useValue: {} }, + ], }).compileComponents(); }) ); diff --git a/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.component.ts b/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.component.ts index 5e307a7d0c..26c90bc243 100644 --- a/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.component.ts +++ b/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.component.ts @@ -13,26 +13,112 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { Component, OnInit } from '@angular/core'; -import { Observable } from 'rxjs'; +import { Component, Input, OnInit } from '@angular/core'; +import { MatDialog } from '@angular/material/dialog'; +import { ActivatedRoute } from '@angular/router'; +import { TranslateService } from '@ngx-translate/core'; +import { Observable, Subscription } from 'rxjs'; +import { first, filter, tap, map } from 'rxjs/operators'; +import { CircleLoadingProgressEnum } from 'src/app/data/enums/circle-loading-progress.enum'; import { Application } from '../../data/interfaces/application'; import { Model } from '../../data/interfaces/model'; import { BreadcrumbsFacade } from '../breadcrumbs/breadcrumbs.facade'; +import { EditDialogComponent } from '../edit-dialog/edit-dialog.component'; +import { ManageAppUsersDialog } from '../manage-app-users-dialog/manage-app-users.component'; @Component({ selector: 'app-breadcrumbs-container', - template: ` `, + template: ` + `, styleUrls: ['./breadcrumbs.container.component.scss'], }) export class BreadcrumbsContainerComponent implements OnInit { app$: Observable; model$: Observable; + modelSelected: boolean; + itemsInProgress$: Observable; + currentUserRole$: Observable; + applications: Application[]; + subs: Subscription; - constructor(private breadcrumbsFacade: BreadcrumbsFacade) {} + @Input() hideControls: boolean; - ngOnInit() { + constructor( + private breadcrumbsFacade: BreadcrumbsFacade, + private translate: TranslateService, + private dialog: MatDialog, + private route: ActivatedRoute + ) {} + + ngOnInit(): void { this.app$ = this.breadcrumbsFacade.app$; + this.subs = this.breadcrumbsFacade.applications$.subscribe(apps => (this.applications = apps)); this.model$ = this.breadcrumbsFacade.model$; + this.modelSelected = !!this.route.snapshot.queryParams.model; + this.currentUserRole$ = this.breadcrumbsFacade.currentUserRole$; + this.itemsInProgress$ = this.breadcrumbsFacade.collectionItems$.pipe( + map(collection => !!collection.find(item => item.status === CircleLoadingProgressEnum.InProgress)) + ); + } + + onAppSettings(app: Application): void { + const fullName = app.owner.firstName + ' ' + app.owner.lastName; + + const dialog = this.dialog.open(EditDialogComponent, { + panelClass: 'custom-mat-dialog', + data: { + type: this.translate.instant('applications.header.title'), + label: this.translate.instant('applications.header.owner'), + errorMsg: this.translate.instant('applications.error_msg'), + entityName: app.name, + ownerName: fullName, + models: this.applications, + }, + }); + dialog + .afterClosed() + .pipe( + first(), + filter(res => res), + tap(res => { + res.update ? this.breadcrumbsFacade.rename(res.name, app) : this.breadcrumbsFacade.delete(app); + }) + ) + .subscribe(); + } + + onUsersList(app: Application): void { + let currentUserId; + let currentUserRole; + + const collection$ = this.breadcrumbsFacade.appUsers$; + + const userSubs = this.breadcrumbsFacade.currentUserId$.subscribe(userId => (currentUserId = userId)); + const userRoleSubs = this.breadcrumbsFacade.currentUserRole$.subscribe(userRole => (currentUserRole = userRole)); + + const dialog = this.dialog.open(ManageAppUsersDialog, { + data: { + collection: collection$, + currentApp: app, + currentUserId: currentUserId, + currentUserRole: currentUserRole, + }, + }); + + const dialogSubs = dialog.afterClosed().subscribe(() => { + userSubs.unsubscribe(); + dialogSubs.unsubscribe(); + userRoleSubs.unsubscribe(); + }); } } diff --git a/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.module.ts b/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.module.ts index 9303d9ac6c..cd0c8bfa87 100644 --- a/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.module.ts +++ b/ui/src/app/features/breadcrumbs.container/breadcrumbs.container.module.ts @@ -17,11 +17,12 @@ import { CommonModule } from '@angular/common'; import { NgModule } from '@angular/core'; import { BreadcrumbsModule } from '../breadcrumbs/breadcrumbs.module'; +import { ManageAppUsersModule } from '../manage-app-users-dialog/manage-app-users.module'; import { BreadcrumbsContainerComponent } from './breadcrumbs.container.component'; @NgModule({ declarations: [BreadcrumbsContainerComponent], exports: [BreadcrumbsContainerComponent], - imports: [CommonModule, BreadcrumbsModule], + imports: [CommonModule, BreadcrumbsModule, ManageAppUsersModule], }) export class BreadcrumbsContainerModule {} diff --git a/ui/src/app/features/breadcrumbs/breadcrumbs.component.html b/ui/src/app/features/breadcrumbs/breadcrumbs.component.html index 2a96c82503..bd29ae0a36 100644 --- a/ui/src/app/features/breadcrumbs/breadcrumbs.component.html +++ b/ui/src/app/features/breadcrumbs/breadcrumbs.component.html @@ -15,28 +15,63 @@ -->
-
- {{ 'breadcrumbs.home' | translate }} -
/
+
+ {{ 'breadcrumbs.home' | translate }} +
>
{{ app.name }} -
/
-
{{ model.name }}
+
>
+
+ {{ model.name }} +
- {{ 'breadcrumbs.home' | translate }} -
/
-
+ {{ 'breadcrumbs.home' | translate }} +
>
+
{{ app.name }}
+ +
+ + + + +
+
diff --git a/ui/src/app/features/breadcrumbs/breadcrumbs.component.scss b/ui/src/app/features/breadcrumbs/breadcrumbs.component.scss index 842cbbad37..de17dcd732 100644 --- a/ui/src/app/features/breadcrumbs/breadcrumbs.component.scss +++ b/ui/src/app/features/breadcrumbs/breadcrumbs.component.scss @@ -14,23 +14,49 @@ * permissions and limitations under the License. */ -@import "mixins.scss"; +@import 'mixins.scss'; +@import 'colors.scss'; +@import 'sizes.scss'; .breadcrumbs { - margin-bottom: 24px; + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 10px; &--container { display: flex; overflow: hidden; .link { - margin-right: 8px; + cursor: pointer; + color: $breadcrumbs-blue; + margin: 0 3px; max-width: 180px; + height: 20px; + font-size: $text-md; @include not-break-string(); + + &.current { + text-decoration: underline; + } } + } - .separator { - margin-right: 8px; + &--actions { + button { + background: none; + border: none; + padding: 0; + &:disabled { + opacity: 0.5; + } + .mat-icon { + width: 23px; + height: 23px; + cursor: pointer; + margin: 5px 10px 0 0; + } } } } diff --git a/ui/src/app/features/breadcrumbs/breadcrumbs.component.spec.ts b/ui/src/app/features/breadcrumbs/breadcrumbs.component.spec.ts index 9f35c9cdf4..0cfea63bfb 100644 --- a/ui/src/app/features/breadcrumbs/breadcrumbs.component.spec.ts +++ b/ui/src/app/features/breadcrumbs/breadcrumbs.component.spec.ts @@ -16,8 +16,10 @@ import { CommonModule } from '@angular/common'; import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; import { MatCardModule } from '@angular/material/card'; +import { MatDialogModule } from '@angular/material/dialog'; import { RouterModule } from '@angular/router'; import { RouterTestingModule } from '@angular/router/testing'; +import { TranslateService } from '@ngx-translate/core'; import { BreadcrumbsComponent } from './breadcrumbs.component'; import { BreadcrumbsFacade } from './breadcrumbs.facade'; @@ -26,20 +28,16 @@ describe('BreadcrumbsComponent', () => { let component: BreadcrumbsComponent; let fixture: ComponentFixture; - beforeEach( - waitForAsync(() => { - TestBed.configureTestingModule({ - declarations: [BreadcrumbsComponent], - providers: [ - { - provide: BreadcrumbsFacade, - useValue: {}, - }, - ], - imports: [CommonModule, RouterModule, RouterTestingModule, MatCardModule], - }).compileComponents(); - }) - ); + beforeEach(waitForAsync(() => { + TestBed.configureTestingModule({ + declarations: [BreadcrumbsComponent], + providers: [ + { provide: BreadcrumbsFacade, useValue: {} }, + { provide: TranslateService, useValue: {} }, + ], + imports: [CommonModule, RouterModule, RouterTestingModule, MatCardModule, MatDialogModule], + }).compileComponents(); + })); beforeEach(() => { fixture = TestBed.createComponent(BreadcrumbsComponent); diff --git a/ui/src/app/features/breadcrumbs/breadcrumbs.component.ts b/ui/src/app/features/breadcrumbs/breadcrumbs.component.ts index 3b5d8cf667..58aea267b9 100644 --- a/ui/src/app/features/breadcrumbs/breadcrumbs.component.ts +++ b/ui/src/app/features/breadcrumbs/breadcrumbs.component.ts @@ -13,11 +13,16 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { Component, Input } from '@angular/core'; +import { Component, EventEmitter, Input, Output } from '@angular/core'; +import { MatDialog } from '@angular/material/dialog'; +import { Router } from '@angular/router'; +import { TranslateService } from '@ngx-translate/core'; +import { Role } from 'src/app/data/enums/role.enum'; import { Routes } from '../../data/enums/routers-url.enum'; import { Application } from '../../data/interfaces/application'; import { Model } from '../../data/interfaces/model'; +import { ConfirmDialogComponent } from '../confirm-dialog/confirm-dialog.component'; @Component({ selector: 'app-breadcrumbs', @@ -27,6 +32,35 @@ import { Model } from '../../data/interfaces/model'; export class BreadcrumbsComponent { routes = Routes; maxNameLength = 30; + role = Role; @Input() model: Model; @Input() app: Application; + @Input() hideControls: boolean; + @Input() modelSelected: boolean; + @Input() itemsInProgress: boolean; + @Input() currentUserRole: string; + + @Output() usersList = new EventEmitter(); + @Output() appSettings = new EventEmitter(); + + constructor(private router: Router, private dialog: MatDialog, private translate: TranslateService) {} + + onNavigate(path: string, id?: string) { + this.itemsInProgress ? this.openDialog(path, id) : this.router.navigate([path], { queryParams: { app: id } }); + } + + openDialog(path: string, id?: string): void { + const dialog = this.dialog.open(ConfirmDialogComponent, { + panelClass: 'custom-mat-dialog', + data: { + title: this.translate.instant('org_users.confirm_dialog.title'), + description: this.translate.instant('org_users.confirm_dialog.confirmation_question'), + }, + }); + + dialog.afterClosed().subscribe(confirm => { + if (!confirm) return; + this.router.navigate([path], { queryParams: { app: id } }); + }); + } } diff --git a/ui/src/app/features/breadcrumbs/breadcrumbs.facade.ts b/ui/src/app/features/breadcrumbs/breadcrumbs.facade.ts index cfc8be13db..887d35f509 100644 --- a/ui/src/app/features/breadcrumbs/breadcrumbs.facade.ts +++ b/ui/src/app/features/breadcrumbs/breadcrumbs.facade.ts @@ -15,21 +15,87 @@ */ import { Injectable } from '@angular/core'; import { Store } from '@ngrx/store'; -import { Observable } from 'rxjs'; +import { Observable, Subscription } from 'rxjs'; +import { Role } from 'src/app/data/enums/role.enum'; +import { AppUser } from 'src/app/data/interfaces/app-user'; +import { deleteUserFromApplication, inviteAppUser, updateAppUserRole } from 'src/app/store/app-user/action'; +import { selectAppUsers, selectAvailableEmails, selectAvailableRoles } from 'src/app/store/app-user/selectors'; +import { updateApplication, deleteApplication } from 'src/app/store/application/action'; +import { selectImageCollection } from 'src/app/store/manage-collectiom/selectors'; +import { selectCurrentUserRole, selectUsers } from 'src/app/store/user/selectors'; +import { selectUserId } from 'src/app/store/userInfo/selectors'; import { Application } from '../../data/interfaces/application'; import { Model } from '../../data/interfaces/model'; import { AppState } from '../../store'; -import { selectCurrentApp } from '../../store/application/selectors'; +import { selectApplications, selectCurrentApp, selectCurrentAppId } from '../../store/application/selectors'; import { selectCurrentModel } from '../../store/model/selectors'; @Injectable() export class BreadcrumbsFacade { app$: Observable; + applications$: Observable; model$: Observable; + selectedId$: Observable; + appUsers$: Observable; + currentUserRole$: Observable; + organizationUsers$: Observable; + availableRoles$: Observable; + availableEmails$: Observable; + currentUserId$: Observable; + selectedId: string | null; + appIdSub: Subscription; + collectionItems$: Observable; constructor(private store: Store) { this.app$ = this.store.select(selectCurrentApp); + this.applications$ = this.store.select(selectApplications); this.model$ = this.store.select(selectCurrentModel); + this.selectedId$ = this.store.select(selectCurrentAppId); + this.appUsers$ = this.store.select(selectAppUsers); + this.currentUserRole$ = this.store.select(selectCurrentUserRole); + this.currentUserId$ = this.store.select(selectUserId); + this.availableEmails$ = this.store.select(selectAvailableEmails); + this.availableRoles$ = this.store.select(selectAvailableRoles); + this.collectionItems$ = this.store.select(selectImageCollection); + } + + rename(name: string, app: Application) { + this.store.dispatch(updateApplication({ name, id: app.id })); + } + + delete(app: Application) { + this.store.dispatch(deleteApplication({ id: app.id })); + } + + updateUserRole(id: string, role: Role, appId: string): void { + this.store.dispatch( + updateAppUserRole({ + applicationId: appId, + user: { + id, + role, + }, + }) + ); + } + + deleteAppUsers(userId: string, appId: string) { + this.store.dispatch( + deleteUserFromApplication({ + applicationId: appId, + userId, + }) + ); + } + + inviteUser(email: string, role: Role, appId: string): void { + this.store.dispatch( + inviteAppUser({ + applicationId: appId, + userEmail: email, + role, + }) + ); } } diff --git a/ui/src/app/features/breadcrumbs/breadcrumbs.module.ts b/ui/src/app/features/breadcrumbs/breadcrumbs.module.ts index be6a4aeff1..b172377b63 100644 --- a/ui/src/app/features/breadcrumbs/breadcrumbs.module.ts +++ b/ui/src/app/features/breadcrumbs/breadcrumbs.module.ts @@ -16,18 +16,31 @@ import { CommonModule } from '@angular/common'; import { NgModule } from '@angular/core'; import { MatCardModule } from '@angular/material/card'; +import { MatIconModule } from '@angular/material/icon'; import { MatTooltipModule } from '@angular/material/tooltip'; import { RouterModule } from '@angular/router'; import { TranslateModule } from '@ngx-translate/core'; import { TruncateModule } from '../../ui/truncate-pipe/truncate.module'; +import { InviteDialogComponent } from '../invite-dialog/invite-dialog.component'; +import { InviteDialogModule } from '../invite-dialog/invite-dialog.module'; import { BreadcrumbsComponent } from './breadcrumbs.component'; import { BreadcrumbsFacade } from './breadcrumbs.facade'; @NgModule({ declarations: [BreadcrumbsComponent], exports: [BreadcrumbsComponent], - imports: [CommonModule, RouterModule, MatCardModule, TruncateModule, MatTooltipModule, TranslateModule], + imports: [ + CommonModule, + RouterModule, + MatCardModule, + TruncateModule, + MatTooltipModule, + TranslateModule, + MatIconModule, + InviteDialogModule, + ], providers: [BreadcrumbsFacade], + entryComponents: [InviteDialogComponent], }) export class BreadcrumbsModule {} diff --git a/ui/src/app/features/change-password-dialog/change-password-dialog.component.html b/ui/src/app/features/change-password-dialog/change-password-dialog.component.html index 4fc14d4830..5af74aae1f 100644 --- a/ui/src/app/features/change-password-dialog/change-password-dialog.component.html +++ b/ui/src/app/features/change-password-dialog/change-password-dialog.component.html @@ -14,67 +14,74 @@ ~ permissions and limitations under the License. --> -

{{ 'toolbar.change_password' | translate }}

+

{{ 'toolbar.change_password' | translate }}

- - {{ 'change_password.old_password' | translate }} - +
+ +
{{ 'registration.password_restriction' | translate }} - +
- +
- - {{ 'change_password.new_password' | translate }} - {{ 'registration.password_restriction' | translate }} - +
+ +
{{ 'registration.password_restriction' | translate }} - +
- +
-
-
- - -
+ diff --git a/ui/src/app/features/change-password-dialog/change-password-dialog.component.scss b/ui/src/app/features/change-password-dialog/change-password-dialog.component.scss index 438348484a..09b5919995 100644 --- a/ui/src/app/features/change-password-dialog/change-password-dialog.component.scss +++ b/ui/src/app/features/change-password-dialog/change-password-dialog.component.scss @@ -13,4 +13,32 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ +@import 'forms.scss'; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'sizes.scss'; +.form-field { + @include form-field($light-gray, $lighter-gray); +} + +.title { + @include dialog-title; +} + +.actions { + width: 100%; + display: flex; + justify-content: space-between; + margin: 2rem 0 1rem; + + .cancel { + @include action-btns(129px, $light-blue, white); + border: 2px solid $light-blue; + } + + .change { + @include action-btns(140px, white, $light-blue); + border: none; + } +} diff --git a/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.html b/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.html new file mode 100644 index 0000000000..aba808fcbe --- /dev/null +++ b/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.html @@ -0,0 +1,33 @@ + + +
+ highlight_off +
+
+ warning_amber +
+
+ +
+
+ diff --git a/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.scss b/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.scss new file mode 100644 index 0000000000..2b004ef4d9 --- /dev/null +++ b/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.scss @@ -0,0 +1,68 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'colors.scss'; +$attention-icon-size: 6rem; + +:host { + position: absolute; + width: 100%; + height: 100%; + left: 0; + top: 0; + display: flex; + flex-direction: row-reverse; + align-items: center; + justify-content: center; + + &.blurred { + backdrop-filter: blur(20px) brightness(0.6); + + &:hover { + opacity: 0.7; + transition: all 0.3s ease-in-out; + } + } + + .image-load-failed, + .image-load-onhold, + .image-load-in-progress { + width: $attention-icon-size; + + mat-icon { + color: $white; + opacity: 0.7; + font-size: $attention-icon-size; + } + } + + .image-load-onhold { + cursor: pointer; + } + + .image-load-in-progress { + display: flex; + flex-direction: row-reverse; + align-items: center; + justify-content: center; + } + + ::ng-deep .mat-progress-spinner circle, + .mat-spinner circle { + stroke: $white; + opacity: 0.7; + } +} diff --git a/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.ts b/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.ts new file mode 100644 index 0000000000..c1730cf674 --- /dev/null +++ b/ui/src/app/features/circle-loading-progress/circle-loading-progress.component.ts @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { ChangeDetectionStrategy, Component, EventEmitter, HostBinding, Input, Output, OnChanges, SimpleChanges } from '@angular/core'; +import { CircleLoadingProgressEnum } from 'src/app/data/enums/circle-loading-progress.enum'; + +@Component({ + selector: 'circle-progress', + templateUrl: './circle-loading-progress.component.html', + styleUrls: ['./circle-loading-progress.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class CircleLoadingProgressComponent implements OnChanges { + @Input() state = CircleLoadingProgressEnum.InProgress; + @Input() error: string; + + @Output() cancel = new EventEmitter(); + + inProgressDiameter = 80; + inProgressStrokeWidth = 4; + + progressEnum = CircleLoadingProgressEnum; + + @HostBinding('class.blurred') isBlurred: boolean = false; + + ngOnChanges(changes: SimpleChanges): void { + if (changes.state?.currentValue) { + this.isBlurred = this.state !== CircleLoadingProgressEnum.Uploaded; + } + } +} diff --git a/ui/src/app/features/circle-loading-progress/circle-loading-progress.module.ts b/ui/src/app/features/circle-loading-progress/circle-loading-progress.module.ts new file mode 100644 index 0000000000..98e90a93c8 --- /dev/null +++ b/ui/src/app/features/circle-loading-progress/circle-loading-progress.module.ts @@ -0,0 +1,28 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { CircleLoadingProgressComponent } from './circle-loading-progress.component'; +import { MatIconModule } from '@angular/material/icon'; +import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; +import { MatTooltipModule } from '@angular/material/tooltip'; + +@NgModule({ + declarations: [CircleLoadingProgressComponent], + exports: [CircleLoadingProgressComponent], + imports: [CommonModule, MatIconModule, MatProgressSpinnerModule, MatTooltipModule], +}) +export class CircleLoadingProgressModule {} diff --git a/ui/src/app/features/collection-manager-subject-left/collection-left-facade.ts b/ui/src/app/features/collection-manager-subject-left/collection-left-facade.ts index cd56f73308..6082f2b7dd 100644 --- a/ui/src/app/features/collection-manager-subject-left/collection-left-facade.ts +++ b/ui/src/app/features/collection-manager-subject-left/collection-left-facade.ts @@ -17,13 +17,9 @@ import { Injectable } from '@angular/core'; import { Store } from '@ngrx/store'; import { Observable } from 'rxjs'; -import { addSubject, loadSubjects, setSelectedSubject } from '../../store/manage-collectiom/action'; -import { - selectAddSubjectPending, - selectCollectionSubject, - selectCollectionSubjects, -} from '../../store/manage-collectiom/selectors'; -import { selectCurrentApiKey } from "../../store/model/selectors"; +import { addSubject, deleteSubject, editSubject, loadSubjects, setSelectedSubject } from '../../store/manage-collectiom/action'; +import { selectAddSubjectPending, selectCollectionSubject, selectCollectionSubjects } from '../../store/manage-collectiom/selectors'; +import { selectCurrentApiKey } from '../../store/model/selectors'; @Injectable() export class CollectionLeftFacade { @@ -39,6 +35,14 @@ export class CollectionLeftFacade { this.apiKey$ = this.store.select(selectCurrentApiKey); } + edit(editName: string, subject: string, apiKey: string): void { + this.store.dispatch(editSubject({ editName, apiKey, subject })); + } + + delete(name: string, apiKey: string): void { + this.store.dispatch(deleteSubject({ name, apiKey })); + } + loadSubjects(apiKey: string): void { this.store.dispatch(loadSubjects({ apiKey })); } diff --git a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left.container.component.ts b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left.container.component.ts index bd8b83b813..76cd7c684d 100644 --- a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left.container.component.ts +++ b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left.container.component.ts @@ -13,43 +13,74 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, OnDestroy, OnInit, Output, EventEmitter } from '@angular/core'; import { Observable } from 'rxjs'; +import { filter, first, map, tap } from 'rxjs/operators'; +import { Subscription } from 'rxjs'; import { CollectionLeftFacade } from './collection-left-facade'; +import { CollectionRightFacade } from '../collection-manager-subject-right/collection-manager-right-facade'; import { CreateDialogComponent } from '../create-dialog/create-dialog.component'; import { TranslateService } from '@ngx-translate/core'; import { MatDialog } from '@angular/material/dialog'; -import { MatListOption } from '@angular/material/list'; +import { ConfirmDialogComponent } from '../confirm-dialog/confirm-dialog.component'; +import { CircleLoadingProgressEnum } from 'src/app/data/enums/circle-loading-progress.enum'; +import { DeleteDialogComponent } from '../delete-dialog/delete-dialog.component'; +import { MergerDialogComponent } from '../merger-dialog/merger-dialog.component'; +import { EditSubjectDialog } from '../edit-subject/edit-subject-dialog.component'; +import { Input } from '@angular/core'; @Component({ selector: 'app-application-list-container', template: ``, changeDetection: ChangeDetectionStrategy.OnPush, }) -export class CollectionManagerSubjectLeftContainerComponent implements OnInit { +export class CollectionManagerSubjectLeftContainerComponent implements OnInit, OnDestroy { currentSubject$: Observable; - subjectsList$: Observable; isPending$: Observable; apiKey$: Observable; + collectionItemsSubs: Subscription; + subjectsSubs: Subscription; + itemsInProgress: boolean; + subjectsList: string[]; + + @Input() isCollectionOnHold: boolean; + @Input() search: string; + @Output() setDefaultMode = new EventEmitter(); private apiKey: string; - constructor(private collectionLeftFacade: CollectionLeftFacade, private translate: TranslateService, private dialog: MatDialog) { - } + constructor( + private collectionLeftFacade: CollectionLeftFacade, + private translate: TranslateService, + private dialog: MatDialog, + private collectionRightFacade: CollectionRightFacade + ) {} ngOnInit(): void { this.currentSubject$ = this.collectionLeftFacade.currentSubject$; - this.subjectsList$ = this.collectionLeftFacade.subjectsList$; this.isPending$ = this.collectionLeftFacade.isPending$; this.apiKey$ = this.collectionLeftFacade.apiKey$; + + this.subjectsSubs = this.collectionLeftFacade.subjectsList$.subscribe(subjects => (this.subjectsList = subjects)); + + this.collectionItemsSubs = this.collectionRightFacade.collectionItems$ + .pipe( + map(collection => collection.filter(item => item.status === CircleLoadingProgressEnum.InProgress)), + tap(collection => (this.itemsInProgress = !!collection.length)) + ) + .subscribe(); } initApiKey(apiKey: string): void { @@ -57,25 +88,118 @@ export class CollectionManagerSubjectLeftContainerComponent implements OnInit { this.collectionLeftFacade.loadSubjects(apiKey); } - addSubject(): void { + delete(name: string): void { + const dialog = this.dialog.open(DeleteDialogComponent, { + panelClass: 'custom-mat-dialog', + data: { + entityType: this.translate.instant('manage_collection.right_side.delete_subject'), + entityName: name, + }, + }); + + dialog + .afterClosed() + .pipe( + first(), + filter(result => result) + ) + .subscribe(() => this.collectionLeftFacade.delete(name, this.apiKey)); + } + + edit(name: string): void { + const dialog = this.dialog.open(EditSubjectDialog, { + panelClass: 'custom-mat-dialog', + data: { + type: this.translate.instant('manage_collection.right_side.edit_subject'), + entityName: name, + }, + }); + + dialog + .afterClosed() + .pipe( + first(), + filter(result => result) + ) + .subscribe(editName => { + this.subjectsList.includes(editName) ? this.merger(editName, name) : this.collectionLeftFacade.edit(editName, name, this.apiKey); + }); + } + + merger(editName: string, name: string): void { + const dialog = this.dialog.open(MergerDialogComponent, { + panelClass: 'custom-mat-dialog', + data: { + entityType: this.translate.instant('manage_collection.right_side.edit_subject'), + }, + }); + + dialog + .afterClosed() + .pipe( + first(), + filter(result => result) + ) + .subscribe(() => this.collectionLeftFacade.edit(editName, name, this.apiKey)); + } + + openCreateDialog(): void { const dialog = this.dialog.open(CreateDialogComponent, { panelClass: 'custom-mat-dialog', data: { entityType: this.translate.instant('manage_collection.left_side.modal_title'), placeholder: this.translate.instant('manage_collection.left_side.subject_name'), + errorMsg: this.translate.instant('manage_collection.error_msg'), + nameList: this.subjectsList, name: '', }, }); const dialogSubscription = dialog.afterClosed().subscribe(name => { - if (name) { - this.collectionLeftFacade.addSubject(name, this.apiKey); - dialogSubscription.unsubscribe(); - } + dialogSubscription.unsubscribe(); + if (!name) return; + this.collectionLeftFacade.addSubject(name, this.apiKey); + }); + } + + addSubject(currentSubject: string): void { + this.itemsInProgress ? this.openDialog(currentSubject, true) : this.openCreateDialog(); + } + + onSelectedSubject(subject: string): void { + this.setDefaultMode.emit(); + + if (this.itemsInProgress) { + this.openDialog(subject); + } else { + this.collectionLeftFacade.onSelectedSubject(subject); + this.collectionRightFacade.loadSubjectMedia(subject); + } + } + + openDialog(subject: string, addSubjectInprogress = false): void { + const dialog = this.dialog.open(ConfirmDialogComponent, { + panelClass: 'custom-mat-dialog', + data: { + title: this.translate.instant('org_users.confirm_dialog.title'), + description: this.translate.instant('org_users.confirm_dialog.confirmation_question'), + }, + }); + + dialog.afterClosed().subscribe(confirm => { + if (!confirm) return; + + this.itemsInProgress = false; + + if (addSubjectInprogress) this.addSubject(subject); + + this.collectionLeftFacade.onSelectedSubject(subject); + this.collectionRightFacade.loadSubjectMedia(subject); }); } - onSelectedSubject(change: MatListOption[]): void { - this.collectionLeftFacade.onSelectedSubject(change[0].value); + ngOnDestroy() { + this.collectionItemsSubs.unsubscribe(); + this.subjectsSubs.unsubscribe(); } } diff --git a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left.module.ts b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left.module.ts index 56f61f07c0..8d5d4d75c1 100644 --- a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left.module.ts +++ b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left.module.ts @@ -21,14 +21,19 @@ import { MatListModule } from '@angular/material/list'; import { TranslateModule } from '@ngx-translate/core'; import { MatButtonModule } from '@angular/material/button'; import { MatIconModule } from '@angular/material/icon'; +import { MatDialogModule } from '@angular/material/dialog'; +import { MatTooltipModule } from '@angular/material/tooltip'; import { AppSearchTableModule } from '../app-search-table/app-search-table.module'; import { CollectionLeftFacade } from './collection-left-facade'; import { SpinnerModule } from '../spinner/spinner.module'; import { TablePipeModule } from '../../ui/search-pipe/table-filter.module'; import { CollectionManagerSubjectLeftContainerComponent } from './collection-manager-subject-left.container.component'; +import { CollectionRightFacade } from '../collection-manager-subject-right/collection-manager-right-facade'; +import { ConfirmDialogComponent } from '../confirm-dialog/confirm-dialog.component'; +import { TruncateModule } from 'src/app/ui/truncate-pipe/truncate.module'; @NgModule({ - declarations: [CollectionManagerSubjectLeftContainerComponent, CollectionManagerSubjectLeftComponent], + declarations: [CollectionManagerSubjectLeftContainerComponent, CollectionManagerSubjectLeftComponent, ConfirmDialogComponent], imports: [ CommonModule, MatListModule, @@ -38,8 +43,11 @@ import { CollectionManagerSubjectLeftContainerComponent } from './collection-man AppSearchTableModule, SpinnerModule, TablePipeModule, + MatDialogModule, + MatTooltipModule, + TruncateModule, ], exports: [CollectionManagerSubjectLeftContainerComponent], - providers: [CollectionLeftFacade], + providers: [CollectionLeftFacade, CollectionRightFacade], }) export class CollectionManagerSubjectLeftModule {} diff --git a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.html b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.html index f3da175d8d..74dbbd92f2 100644 --- a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.html +++ b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.html @@ -14,36 +14,56 @@ ~ permissions and limitations under the License. -->
-
- -
-
- - -
- - - -
- -
{{ subject }}
+
    +
  • +
    + {{ + subject | truncate: 40 + }} + +
    + + + +
    - - +
    • +
+
+ +
diff --git a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.scss b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.scss index fb92433dbc..da6a3d7ebe 100644 --- a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.scss +++ b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.scss @@ -15,62 +15,99 @@ */ @import 'colors.scss'; @import 'mixins.scss'; +@import 'sizes.scss'; -.main { - .add-subject { - padding-bottom: 16px; - border-bottom: 1px solid $light-gray-special; +$blue: #516581; - .mat-flat-button { - height: 43px; - width: 100%; - } - } - - .search { - margin-top: 15px; - } +.main { + height: 100%; + display: flex; + flex-direction: column; .subjects { - height: calc(100vh - 438px); - min-height: 300px; + height: 90%; overflow-y: auto; + .select { + border: none; + padding: 0; + margin: 0; + } + &--spinner { position: absolute; left: 50%; transform: translateX(-50%); } - .section { + .option { border-bottom: 1px solid $special-gray; - border-radius: 1px; - cursor: pointer; + list-style: none; + padding: 0.5rem 0.7rem; + &[selected='true'], &:hover { - border-bottom: 1px solid $primary; - } - - div { - padding: 0 !important; + background: $bg-gray; + border-radius: 8px; } .avatar { - margin-right: 13px; + cursor: pointer; display: flex; + justify-content: space-between; align-items: center; - mat-icon { - flex-shrink: 0; + .actions { + display: flex; + justify-content: space-between; + width: 45px; + + button { + cursor: pointer; + display: flex; + justify-content: center; + align-items: center; + width: 20px; + height: 20px; + + .mat-icon { + display: flex; + justify-content: center; + align-items: center; + flex-shrink: 0; + color: $orange; + + &.del-icon { + @include sm-icon-size(17px); + } + + &.edit-icon { + @include sm-icon-size(14px); + } + } + } } .name { - font-size: 18px; - color: $black; - margin-left: 13px; + font-size: $text-sm; + color: $blue; @include not-break-string(); } } } } + + .btn-container { + height: 10%; + width: 100%; + display: flex; + + button { + @include action-btns(100%, white, $light-blue); + height: 35px; + font-size: $text-sm; + margin: auto 0 10px; + border-radius: 10px; + } + } } diff --git a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.ts b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.ts index 77336403b0..50d9a2f4e1 100644 --- a/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.ts +++ b/ui/src/app/features/collection-manager-subject-left/collection-manager-subject-left/collection-manager-subject-left.component.ts @@ -14,7 +14,6 @@ * permissions and limitations under the License. */ import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output } from '@angular/core'; -import { MatListOption } from '@angular/material/list'; @Component({ selector: 'app-collection-manager-subject-left', @@ -25,17 +24,25 @@ import { MatListOption } from '@angular/material/list'; export class CollectionManagerSubjectLeftComponent { @Input() subjectsList: string[]; @Input() currentSubject: string; + @Input() search: string; @Input() isPending: boolean; + @Input() isCollectionOnHold: boolean; @Input() set apiKey(value: string) { - if(!!value) this.initApiKey.emit(value) - }; - @Output() addSubject = new EventEmitter(); - @Output() selectedSubject = new EventEmitter(); - @Output() initApiKey = new EventEmitter(); + if (!!value) this.initApiKey.emit(value); + } - search = ''; + @Output() deleteSubject = new EventEmitter(); + @Output() editSubject = new EventEmitter(); + @Output() addSubject = new EventEmitter(); + @Output() selectedSubject = new EventEmitter(); + @Output() initApiKey = new EventEmitter(); - onSearch(event: string) { + onSearch(event: string): void { this.search = event; } + + onSelectedSubject(subject: string): void { + if (this.currentSubject === subject) return; + this.selectedSubject.emit(subject); + } } diff --git a/ui/src/app/features/collection-manager-subject-right/collection-manager-right-facade.ts b/ui/src/app/features/collection-manager-subject-right/collection-manager-right-facade.ts index 3944c66656..b73dbb40bf 100644 --- a/ui/src/app/features/collection-manager-subject-right/collection-manager-right-facade.ts +++ b/ui/src/app/features/collection-manager-subject-right/collection-manager-right-facade.ts @@ -17,30 +17,104 @@ import { Injectable } from '@angular/core'; import { Store } from '@ngrx/store'; import { Observable } from 'rxjs'; -import { selectCurrentApiKey } from '../../store/model/selectors'; -import { selectAddSubjectPending, selectCollectionSubject, selectCollectionSubjects } from '../../store/manage-collectiom/selectors'; -import { deleteSubject, editSubject } from '../../store/manage-collectiom/action'; +import { selectCurrentApiKey, selectCurrentModel } from '../../store/model/selectors'; +import { + selectAddSubjectPending, + selectCollectionSubject, + selectCollectionSubjects, + selectImageCollection, + selectImageCollectionPending, + selectSubjectMode, +} from '../../store/manage-collectiom/selectors'; +import { + deleteItemFromUploadOrder, + deleteSelectedExamples, + deleteSubjectExample, + getSubjectExamples, + getSubjectMediaNextPage, + readImageFiles, + resetSubjectExamples, + setSubjectMode, + startUploadImageOrder, + toggleExampleSelection, + uploadImage, +} from '../../store/manage-collectiom/action'; +import { CollectionItem } from 'src/app/data/interfaces/collection'; +import { SubjectModeEnum } from 'src/app/data/enums/subject-mode.enum'; +import { filter, map } from 'rxjs/operators'; +import { selectMaxFileSize } from 'src/app/store/image-size/selectors'; +import { MaxImageSize } from 'src/app/data/interfaces/size.interface'; @Injectable() export class CollectionRightFacade { subjects$: Observable; subject$: Observable; + defaultSubject$: Observable; apiKey$: Observable; + collectionItems$: Observable; isPending$: Observable; + isCollectionPending$: Observable; + subjectMode$: Observable; + currentModelName$: Observable; + maxBodySize$: Observable; constructor(private store: Store) { + this.maxBodySize$ = this.store.select(selectMaxFileSize); + this.defaultSubject$ = this.store.select(selectCollectionSubject); this.subjects$ = this.store.select(selectCollectionSubjects); this.subject$ = this.store.select(selectCollectionSubject); this.apiKey$ = this.store.select(selectCurrentApiKey); + this.collectionItems$ = this.store.select(selectImageCollection); this.isPending$ = this.store.select(selectAddSubjectPending); + this.isCollectionPending$ = this.store.select(selectImageCollectionPending); + this.subjectMode$ = this.store.select(selectSubjectMode); + this.currentModelName$ = this.store.select(selectCurrentModel).pipe( + filter(model => !!model), + map(model => model.name) + ); } + loadSubjectMedia(subject: string): void { + this.store.dispatch(getSubjectExamples({ subject })); + } + + loadNextPage(subject: string, page: number, totalPages: number): void { + this.store.dispatch(getSubjectMediaNextPage({ subject, page, totalPages })); + } + + restartUploading(): void { + this.store.dispatch(startUploadImageOrder()); + } + + addImageFilesToCollection(fileDescriptors: File[]): void { + this.store.dispatch(readImageFiles({ fileDescriptors })); + } + + uploadImage(item: CollectionItem): void { + this.store.dispatch(uploadImage({ item })); + } + + deleteSubjectExample(item: CollectionItem): void { + this.store.dispatch(deleteSubjectExample({ item })); + } + + deleteItemFromUploadOrder(item: CollectionItem): void { + this.store.dispatch(deleteItemFromUploadOrder({ item })); + } + + resetSubjectExamples(): void { + this.store.dispatch(resetSubjectExamples()); + } + + setSubjectMode(mode: SubjectModeEnum): void { + this.store.dispatch(setSubjectMode({ mode })); + } - edit(editName: string, subject: string, apiKey: string): void { - this.store.dispatch(editSubject({ editName, apiKey, subject })); + selectSubjectExample(item: CollectionItem): void { + this.store.dispatch(toggleExampleSelection({ item })); } - delete(name: string, apiKey: string): void { - this.store.dispatch(deleteSubject({ name, apiKey })); + deleteSelectedExamples(ids: string[]): void { + this.store.dispatch(deleteSelectedExamples({ ids })); } } diff --git a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right.container.component.ts b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right.container.component.ts index 2acbbfbf3c..22aa6290fa 100644 --- a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right.container.component.ts +++ b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right.container.component.ts @@ -13,105 +13,102 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; -import { Observable } from 'rxjs'; +import { ChangeDetectionStrategy, Component, EventEmitter, OnDestroy, OnInit, Output } from '@angular/core'; +import { Observable, Subscription } from 'rxjs'; -import { TranslateService } from '@ngx-translate/core'; -import { MatDialog } from '@angular/material/dialog'; import { CollectionRightFacade } from './collection-manager-right-facade'; -import { filter, first, tap } from 'rxjs/operators'; -import { EditDialogComponent } from '../edit-dialog/edit-dialog.component'; -import { DeleteDialogComponent } from '../delete-dialog/delete-dialog.component'; -import { MergerDialogComponent } from '../merger-dialog/merger-dialog.component'; +import { CircleLoadingProgressEnum } from 'src/app/data/enums/circle-loading-progress.enum'; +import { SubjectModeEnum } from 'src/app/data/enums/subject-mode.enum'; +import { CollectionItem } from 'src/app/data/interfaces/collection'; +import { tap } from 'rxjs/operators'; +import { SnackBarService } from '../snackbar/snackbar.service'; @Component({ selector: 'app-application-right-container', template: ``, changeDetection: ChangeDetectionStrategy.OnPush, }) -export class CollectionManagerSubjectRightContainerComponent implements OnInit { - subjects$: Observable; +export class CollectionManagerSubjectRightContainerComponent implements OnInit, OnDestroy { + defaultSubject$: Observable; subject$: Observable; isPending$: Observable; + isCollectionPending$: Observable; apiKey$: Observable; + collectionItems$: Observable; + mode$: Observable; + apiKeyInitSubscription: Subscription; - private subjectList: string[]; + @Output() setDefaultMode = new EventEmitter(); private apiKey: string; - constructor(private collectionRightFacade: CollectionRightFacade, private translate: TranslateService, private dialog: MatDialog) { - } + constructor(private collectionRightFacade: CollectionRightFacade, private snackBarService: SnackBarService) {} ngOnInit(): void { - this.subjects$ = this.collectionRightFacade.subjects$.pipe(tap(subjects => (this.subjectList = subjects))); this.subject$ = this.collectionRightFacade.subject$; this.apiKey$ = this.collectionRightFacade.apiKey$; + this.collectionItems$ = this.collectionRightFacade.collectionItems$; this.isPending$ = this.collectionRightFacade.isPending$; + this.isCollectionPending$ = this.collectionRightFacade.isCollectionPending$; + this.mode$ = this.collectionRightFacade.subjectMode$; + + this.defaultSubject$ = this.collectionRightFacade.defaultSubject$.pipe( + tap(subject => this.collectionRightFacade.loadSubjectMedia(subject)) + ); } initApiKey(apiKey: string): void { this.apiKey = apiKey; } - delete(name: string): void { - const dialog = this.dialog.open(DeleteDialogComponent, { - panelClass: 'custom-mat-dialog', - data: { - entityType: this.translate.instant('manage_collection.right_side.delete_subject'), - entityName: name, - }, - }); - - dialog - .afterClosed() - .pipe( - first(), - filter(result => result) - ) - .subscribe(() => this.collectionRightFacade.delete(name, this.apiKey)); + loadMore(item: CollectionItem): void { + this.collectionRightFacade.loadNextPage(item['subject'], item['page'], item['totalPages']); + } + + restartUploading(): void { + this.collectionRightFacade.restartUploading(); + } + + readFiles(fileList: File[]): void { + this.setDefaultMode.emit(); + const fileBodySize = fileList.map(item => item.size).reduce((previousValue, currentValue) => previousValue + currentValue); + + this.collectionRightFacade.addImageFilesToCollection(fileList); + } + + deleteItem(item: CollectionItem): void { + if (item.status === CircleLoadingProgressEnum.Uploaded) { + this.collectionRightFacade.deleteSubjectExample(item); + + return; + } + + this.collectionRightFacade.deleteItemFromUploadOrder(item); + } + + cancelUploadItem(item: CollectionItem): void { + this.collectionRightFacade.deleteItemFromUploadOrder(item); } - edit(name: string): void { - const dialog = this.dialog.open(EditDialogComponent, { - panelClass: 'custom-mat-dialog', - data: { - entityType: this.translate.instant('manage_collection.right_side.edit_subject'), - entityName: name, - }, - }); - - dialog - .afterClosed() - .pipe( - first(), - filter(result => result) - ) - .subscribe(editName => - this.subjectList.includes(editName) ? this.merger(editName, name) : this.collectionRightFacade.edit(editName, name, this.apiKey) - ); + selectExample(item: CollectionItem): void { + this.collectionRightFacade.selectSubjectExample(item); } - merger(editName: string, name: string): void { - const dialog = this.dialog.open(MergerDialogComponent, { - panelClass: 'custom-mat-dialog', - data: { - entityType: this.translate.instant('manage_collection.right_side.edit_subject'), - }, - }); - - dialog - .afterClosed() - .pipe( - first(), - filter(result => result) - ) - .subscribe(() => this.collectionRightFacade.edit(editName, name, this.apiKey)); + ngOnDestroy(): void { + this.collectionRightFacade.resetSubjectExamples(); } } diff --git a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right.module.ts b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right.module.ts index d9359db1ad..f2e91db38d 100644 --- a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right.module.ts +++ b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right.module.ts @@ -22,11 +22,26 @@ import { SpinnerModule } from '../spinner/spinner.module'; import { MatIconModule } from '@angular/material/icon'; import { MatButtonModule } from '@angular/material/button'; import { TranslateModule } from '@ngx-translate/core'; +import { DragNDropModule } from '../drag-n-drop/drag-n-drop.module'; +import { ImageHolderCollectionModule } from '../image-holder-collection/image-holder-collection.module'; + +import { InfiniteScrollModule } from 'ngx-infinite-scroll'; +import { IntersectionObserverModule } from '@ng-web-apis/intersection-observer'; @NgModule({ declarations: [CollectionManagerSubjectRightContainerComponent, CollectionManagerSubjectRightComponent], exports: [CollectionManagerSubjectRightContainerComponent], - imports: [CommonModule, SpinnerModule, MatIconModule, MatButtonModule, TranslateModule], + imports: [ + CommonModule, + SpinnerModule, + MatIconModule, + MatButtonModule, + TranslateModule, + DragNDropModule, + ImageHolderCollectionModule, + InfiniteScrollModule, + IntersectionObserverModule, + ], providers: [CollectionRightFacade], }) export class CollectionManagerSubjectRightModule {} diff --git a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.html b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.html index a7add45a98..6b993e3f8c 100644 --- a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.html +++ b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.html @@ -13,24 +13,55 @@ ~ or implied. See the License for the specific language governing ~ permissions and limitations under the License. --> -
- - -
-
-
{{ "manage_collection.right_side.profile" | translate }} {{ subject }}
-
- - +
+ + +
+ {{ defaultSubject }} + + {{ totalElements }} {{ 'common.capital.images' | translate }} + +
+
+ + +
+
+ +
+ +
-
{{ 'manage_collection.right_side.examples' | translate }}
-
+
+ + + + + + +
{{ 'manage_collection.right_side.empty_message' | translate }}
+ diff --git a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.scss b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.scss index 7b4d7df297..19f4ac57d8 100644 --- a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.scss +++ b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.scss @@ -13,42 +13,51 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ - -@import 'mixins.scss'; @import 'colors.scss'; @import 'mixins.scss'; +@import 'sizes.scss'; -.profile { - &--spinner { - position: absolute; - left: 50%; - transform: translateX(-50%); - } +$container-top-content-height: 212px; + +.collection-right-container { + height: 98%; + padding-right: 1.5rem; + overflow-y: scroll; + position: relative; &--header { - &_profile { - display: flex; - justify-content: space-between; - align-items: center; + position: sticky; + top: 0; + z-index: 10; + background: $lighter-blue; - .name { - @include not-break-string(); - font-weight: 600; - font-size: 22px; - } + border-bottom: 1px solid $pale-blue; + margin-bottom: 28px; + color: $dark-blue; + .title { + font-size: $text-md; + font-weight: bold; + line-height: 31px; + } + .count { + float: right; + font-size: $text-md; + line-height: 31px; + } + } +} - .buttons { - display: flex; +.drag-and-drop-container { + margin-bottom: 1rem; +} - .mat-icon { - @include md-icon-size(); - } - } - } +.image-holder-collection-container { + padding-top: 1rem; + height: calc(100% - #{$container-top-content-height}); + width: 100%; - &_examples { - font-size: 14px; - color: $light-gray; - } + &--spinner { + display: flex; + justify-content: center; } } diff --git a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.ts b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.ts index 183f21aa73..1a5bc7d19f 100644 --- a/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.ts +++ b/ui/src/app/features/collection-manager-subject-right/collection-manager-subject-right/collection-manager-subject-right.component.ts @@ -13,23 +13,95 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { Component, EventEmitter, Input, Output } from '@angular/core'; +import { Component, EventEmitter, Input, OnChanges, Output, SimpleChanges } from '@angular/core'; +import { CircleLoadingProgressEnum } from 'src/app/data/enums/circle-loading-progress.enum'; +import { SubjectModeEnum } from 'src/app/data/enums/subject-mode.enum'; +import { CollectionItem } from 'src/app/data/interfaces/collection'; @Component({ selector: 'app-collection-manager-subject-right', templateUrl: './collection-manager-subject-right.component.html', styleUrls: ['./collection-manager-subject-right.component.scss'], }) -export class CollectionManagerSubjectRightComponent { +export class CollectionManagerSubjectRightComponent implements OnChanges { + scrollWindow: boolean = false; + prevItemCollection: CollectionItem[] = []; + subjectModeEnum = SubjectModeEnum; + uploadedExamples: CollectionItem[] = []; + isCollectionOnHold: boolean = false; + totalElements: number = 0; + intersectionObserverEvent: IntersectionObserverEntry[]; + @Input() isPending: boolean; + @Input() isCollectionPending: boolean; @Input() subject: string; @Input() subjects: string[]; @Input() set apiKey(value: string) { - if(!!value) this.initApiKey.emit(value) - }; - @Output() deleteSubject = new EventEmitter(); - @Output() editSubject = new EventEmitter(); + if (!!value) this.initApiKey.emit(value); + } + @Input() collectionItems: CollectionItem[]; + @Input() mode = SubjectModeEnum.Default; + @Input() defaultSubject: string; + @Output() initApiKey = new EventEmitter(); + @Output() readFiles = new EventEmitter(); + @Output() deleteItem = new EventEmitter(); + @Output() cancelUploadItem = new EventEmitter(); + @Output() selectExample = new EventEmitter(); + @Output() loadMore = new EventEmitter(); + @Output() restartUploading = new EventEmitter(); + + onIntersection(event: IntersectionObserverEntry[]) { + if (event) { + this.intersectionObserverEvent = event; + const lastElement = event[event.length - 1]; + if ( + lastElement.target.scrollHeight < lastElement.rootBounds.height || + lastElement.target.scrollWidth < lastElement.rootBounds.width + ) { + if (lastElement.intersectionRatio !== 1 && lastElement.rootBounds) return; + if (this.collectionItems.length) this.onScrollDown(); + } + } + } + + ngOnChanges(changes: SimpleChanges) { + const change = changes['collectionItems']; + if (change && change['currentValue'] !== change['previousValue']) { + const collectionOnHold = this.collectionItems.filter(item => item.status === CircleLoadingProgressEnum.OnHold); + + this.isCollectionOnHold = !!collectionOnHold.length; + + const itemInProgress = this.collectionItems.find(item => item.status === CircleLoadingProgressEnum.InProgress); + + if (this.isCollectionOnHold && !itemInProgress) { + this.restartUploading.emit(); + } + + const examples = this.collectionItems.filter( + item => item['totalElements'] === undefined && item.status === CircleLoadingProgressEnum.Uploaded + ); + + this.uploadedExamples = this.collectionItems.filter(item => item.status === CircleLoadingProgressEnum.Uploaded); + + this.uploadedExamples.length && this.collectionItems[0]['totalElements'] + ? (this.totalElements = examples.length + this.collectionItems[0]['totalElements']) + : (this.totalElements = examples.length || 0); + + this.onIntersection(this.intersectionObserverEvent); + } + } + + onScrollDown(): void { + const lastItem = this.uploadedExamples[this.uploadedExamples.length - 1]; + if (lastItem) { + const nextPage = lastItem['page'] + 1; + const totalPages = lastItem['totalPages']; - constructor() {} + if (totalPages !== nextPage && !isNaN(nextPage)) { + this.prevItemCollection = this.collectionItems; + this.loadMore.emit(lastItem); + } + } + } } diff --git a/ui/src/app/features/collection-manager/collection-manager.component.html b/ui/src/app/features/collection-manager/collection-manager.component.html new file mode 100644 index 0000000000..16355a971a --- /dev/null +++ b/ui/src/app/features/collection-manager/collection-manager.component.html @@ -0,0 +1,68 @@ + +
+ +
+

{{ 'manage_collection.header' | translate }}

+ +
+ + +
+ +
+ + + + + + + +
+
+ +
+
+ +
+ +
+ +
+
+ +
diff --git a/ui/src/app/features/collection-manager/collection-manager.component.scss b/ui/src/app/features/collection-manager/collection-manager.component.scss new file mode 100644 index 0000000000..c0c695966b --- /dev/null +++ b/ui/src/app/features/collection-manager/collection-manager.component.scss @@ -0,0 +1,139 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +@import 'media.scss'; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'sizes.scss'; + +.collection { + height: 95%; + + .content { + height: 100%; + background: $lighter-blue; + padding: 1rem !important; + + &--header { + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid $pale-blue; + margin-bottom: 1rem; + padding-bottom: 1rem; + + .header { + color: $dark-blue; + font-size: $text-lg; + font-weight: bold; + margin: 0; + } + + .form-group { + margin-left: 50px; + width: 40%; + min-width: 300px; + border-bottom: 1px solid $pale-blue; + position: relative; + + .mat-icon { + position: absolute; + top: 5px; + right: 5px; + @include sm-icon-size(21px); + } + + input { + width: 100%; + height: 30px; + background: none; + border: none; + outline: none; + font-size: $text-sm; + + &::placeholder { + font-size: $text-sm; + } + } + } + + .container { + padding: 0; + display: flex; + justify-content: flex-end; + + &--btn { + &.select-btn, + &.cancel-btn { + @include action-btns(140px, $light-blue, none); + font-size: $text-sm; + height: 35px; + border-radius: 8px; + border: 2px solid $light-blue; + + &:disabled { + background: none; + border: 2px solid $gray; + color: $gray; + } + } + + &.del-btn { + @include action-btns(140px, $orange, none); + border: 2px solid $orange; + height: 35px; + font-size: $text-sm; + border-radius: 8px; + + &:disabled { + background: none; + border: 2px solid $gray; + color: $gray; + } + } + + &[disabled] { + cursor: auto; + opacity: 0.6; + } + + &.cancel-btn { + margin: 0 20px 0 40px; + } + } + } + } + + &--container { + height: calc(100% - 70px); + padding: 0 !important; + display: flex; + + &_subjects-list { + padding-right: 30px; + border-right: 1px solid grey; + width: 25%; + min-width: 200px; + max-width: 300px; + } + + &_profile { + overflow-y: hidden; + margin-left: 30px; + width: 100%; + } + } + } +} diff --git a/ui/src/app/features/collection-manager/collection-manager.components.ts b/ui/src/app/features/collection-manager/collection-manager.components.ts new file mode 100644 index 0000000000..69594031d2 --- /dev/null +++ b/ui/src/app/features/collection-manager/collection-manager.components.ts @@ -0,0 +1,93 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { MatDialog } from '@angular/material/dialog'; +import { TranslateService } from '@ngx-translate/core'; +import { Subscription } from 'rxjs'; +import { filter, first, map, tap } from 'rxjs/operators'; +import { CircleLoadingProgressEnum } from 'src/app/data/enums/circle-loading-progress.enum'; +import { SubjectModeEnum } from 'src/app/data/enums/subject-mode.enum'; +import { CollectionRightFacade } from '../collection-manager-subject-right/collection-manager-right-facade'; +import { DeleteDialogComponent } from '../delete-dialog/delete-dialog.component'; + +@Component({ + selector: 'collection-manager', + templateUrl: './collection-manager.component.html', + styleUrls: ['./collection-manager.component.scss'], +}) +export class CollectionManagerComponent implements OnInit, OnDestroy { + subjectModeEnum = SubjectModeEnum; + mode = SubjectModeEnum.Default; + + subs: Subscription; + modelSubs: Subscription; + + isCollectionOnHold: boolean; + selectedIds: string[]; + currentModelName: string; + search: string = ''; + + constructor(private collectionRightFacade: CollectionRightFacade, private translate: TranslateService, private dialog: MatDialog) {} + + ngOnInit(): void { + this.modelSubs = this.collectionRightFacade.currentModelName$ + .pipe( + filter(val => !!val), + tap(val => (this.currentModelName = val)) + ) + .subscribe(); + + this.subs = this.collectionRightFacade.collectionItems$ + .pipe( + tap(items => (this.isCollectionOnHold = !!items.filter(item => item.status === CircleLoadingProgressEnum.OnHold).length)), + map(items => (this.selectedIds = items.filter(item => item.isSelected).map(item => item.id))) + ) + .subscribe(); + } + + setBulkSelectMode(): void { + this.mode = SubjectModeEnum.BulkSelect; + this.collectionRightFacade.setSubjectMode(this.mode); + } + + setDefaultMode(): void { + this.mode = SubjectModeEnum.Default; + this.collectionRightFacade.setSubjectMode(this.mode); + } + + deleteSelectedExamples(): void { + const dialog = this.dialog.open(DeleteDialogComponent, { + panelClass: 'custom-mat-dialog', + data: { + entityType: this.translate.instant('manage_collection.right_side.modal_bulk-delete_type'), + }, + }); + + dialog + .afterClosed() + .pipe( + first(), + filter(result => result) + ) + .subscribe(() => this.collectionRightFacade.deleteSelectedExamples(this.selectedIds)); + } + + ngOnDestroy(): void { + this.subs.unsubscribe(); + this.modelSubs.unsubscribe(); + } +} diff --git a/ui/src/app/features/collection-manager/collection-manager.module.ts b/ui/src/app/features/collection-manager/collection-manager.module.ts new file mode 100644 index 0000000000..f44a559ed4 --- /dev/null +++ b/ui/src/app/features/collection-manager/collection-manager.module.ts @@ -0,0 +1,44 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { CommonModule } from '@angular/common'; +import { NgModule } from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { MatCardModule } from '@angular/material/card'; +import { MatIconModule } from '@angular/material/icon'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { TranslateModule } from '@ngx-translate/core'; +import { TruncateModule } from 'src/app/ui/truncate-pipe/truncate.module'; +import { CollectionManagerSubjectLeftModule } from '../collection-manager-subject-left/collection-manager-subject-left.module'; +import { CollectionManagerSubjectRightModule } from '../collection-manager-subject-right/collection-manager-subject-right.module'; +import { CollectionManagerComponent } from './collection-manager.components'; + +@NgModule({ + declarations: [CollectionManagerComponent], + imports: [ + CommonModule, + MatCardModule, + MatTooltipModule, + TranslateModule, + CollectionManagerSubjectLeftModule, + CollectionManagerSubjectRightModule, + FormsModule, + MatIconModule, + TruncateModule, + ], + exports: [CollectionManagerComponent], +}) +export class CollectionManagerModule {} diff --git a/ui/src/app/features/confirm-dialog/confirm-dialog.component.html b/ui/src/app/features/confirm-dialog/confirm-dialog.component.html new file mode 100644 index 0000000000..ee39caf75b --- /dev/null +++ b/ui/src/app/features/confirm-dialog/confirm-dialog.component.html @@ -0,0 +1,30 @@ + +
+

{{ data.title }}

+
+

{{ data.description }}

+
+
+ + +
+ +
diff --git a/java/admin/src/main/java/com/exadel/frs/repository/AppModelRepository.java b/ui/src/app/features/confirm-dialog/confirm-dialog.component.scss similarity index 64% rename from java/admin/src/main/java/com/exadel/frs/repository/AppModelRepository.java rename to ui/src/app/features/confirm-dialog/confirm-dialog.component.scss index d4d89c1b0c..599f7ade74 100644 --- a/java/admin/src/main/java/com/exadel/frs/repository/AppModelRepository.java +++ b/ui/src/app/features/confirm-dialog/confirm-dialog.component.scss @@ -1,25 +1,36 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.repository; - -import com.exadel.frs.commonservice.entity.AppModel; -import com.exadel.frs.commonservice.entity.AppModelId; -import org.springframework.data.jpa.repository.JpaRepository; - -public interface AppModelRepository extends JpaRepository { - -} +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'mixins.scss'; + +.dialog { + &--title { + @include dialog-title; + margin: 0; + } + &--description { + margin: 20px 0; + } +} + +.confirm-btn { + @include action-btns(100px, $white, $light-blue); +} + +.cancel-btn { + @include action-btns(100px, $light-blue, $white); + border: 2px solid $light-blue; +} diff --git a/ui/src/app/features/confirm-dialog/confirm-dialog.component.ts b/ui/src/app/features/confirm-dialog/confirm-dialog.component.ts new file mode 100644 index 0000000000..c9d1e18bde --- /dev/null +++ b/ui/src/app/features/confirm-dialog/confirm-dialog.component.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; + +@Component({ + selector: 'app-confirm-dialog', + templateUrl: './confirm-dialog.component.html', + styleUrls: ['./confirm-dialog.component.scss'], +}) +export class ConfirmDialogComponent { + constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any) {} +} diff --git a/ui/src/app/features/create-application/application/application-name.component.html b/ui/src/app/features/create-application/application/application-name.component.html new file mode 100644 index 0000000000..86adacdb91 --- /dev/null +++ b/ui/src/app/features/create-application/application/application-name.component.html @@ -0,0 +1,49 @@ + +
+

+ {{ 'create_application_wizard.application.title' | translate }} +

+ +

+ {{ 'create_application_wizard.application.info_about_application' | translate }} +

+ +

+ {{ 'create_application_wizard.application.application_name_question' | translate }} +

+ +
+ + + + {{ 'create_application_wizard.application.error_msg' | translate }} + + + +
+ +
+
+
diff --git a/ui/src/app/features/create-application/application/application-name.component.ts b/ui/src/app/features/create-application/application/application-name.component.ts new file mode 100644 index 0000000000..f14e88d642 --- /dev/null +++ b/ui/src/app/features/create-application/application/application-name.component.ts @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { Component, EventEmitter, Input, OnDestroy, OnInit, Output, ViewChild } from '@angular/core'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { MatStepper } from '@angular/material/stepper'; +import { Subscription } from 'rxjs'; + +enum DefaultAppName { + DemoApp = 'Demo app', +} + +@Component({ + selector: 'application-name', + templateUrl: './application-name.component.html', + styleUrls: ['../shared-styles.component.scss'], +}) + +export class ApplicationNameComponent implements OnInit, OnDestroy { + @Output() applicationName = new EventEmitter<{ applicationName: string }>(); + @Input() stepper: MatStepper; + application: FormGroup; + valueSubs: Subscription; + isValid: boolean = true; + + constructor(private fb: FormBuilder) {} + + ngOnInit(): void { + this.application = this.fb.group({ + applicationName: ['', Validators.required], + }); + + this.valueSubs = this.application.get('applicationName').valueChanges.subscribe(val => { + if (val === DefaultAppName.DemoApp) { + this.isValid = false; + return; + } + + this.isValid = true; + }); + } + + onNext(): void { + if (!this.isValid) return; + this.stepper.next(); + this.applicationName.emit(this.application.value.applicationName); + } + + ngOnDestroy(): void { + this.valueSubs.unsubscribe(); + } +} diff --git a/ui/src/app/features/create-application/create-application.component.html b/ui/src/app/features/create-application/create-application.component.html new file mode 100644 index 0000000000..aefa247a65 --- /dev/null +++ b/ui/src/app/features/create-application/create-application.component.html @@ -0,0 +1,37 @@ + +
+ + +
+
+ +
+
+ + + +
+
+ +
+
+ +
+
+ + +
\ No newline at end of file diff --git a/ui/src/app/features/create-application/create-application.component.scss b/ui/src/app/features/create-application/create-application.component.scss new file mode 100644 index 0000000000..99c9961611 --- /dev/null +++ b/ui/src/app/features/create-application/create-application.component.scss @@ -0,0 +1,68 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'colors.scss'; +@import 'sizes.scss'; + +$left-position: 120px; + +.container { + position: absolute; + top: 95px; + left: 0; +} + +.horizontal-stepper { + background: none; +} + +.application__container, +.service__container { + height: calc(100vh - #{$footer-header-height}) !important; + display: flex; + align-items: center; + justify-content: center; +} + +.service__container { + display: flex; + justify-content: flex-start; + align-items: center; + width: 100vw; + + &--create { + height: calc(100vh - #{$footer-header-height}) !important; + width: calc((100vw - #{$left-position}) / 2); + left: $left-position; + bottom: 50px; + position: absolute; + display: flex; + align-items: center; + } + + &--info { + height: calc(100vh - #{$footer-header-height}) !important; + width: calc((100vw - #{$left-position}) / 2); + position: absolute; + right: 0; + padding-left: 115px; + padding-right: 30px; + background: linear-gradient($primary 0%, $primary-lighter 100%); + display: flex; + justify-content: flex-start; + align-items: center; + } +} diff --git a/ui/src/app/features/create-application/create-application.component.ts b/ui/src/app/features/create-application/create-application.component.ts new file mode 100644 index 0000000000..20ac9775dc --- /dev/null +++ b/ui/src/app/features/create-application/create-application.component.ts @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { Component } from '@angular/core'; +import { CreateApplicationFacade } from './create-application.facade'; + +@Component({ + selector: 'app-create-application', + templateUrl: './create-application.component.html', + styleUrls: ['./create-application.component.scss'], +}) +export class CreateApplicationComponent { + applicationName: string; + + constructor(private appFacade: CreateApplicationFacade) {} + + getApplicationName(name: string) { + this.applicationName = name; + this.appFacade.createApplication(this.applicationName); + } +} diff --git a/ui/src/app/features/create-application/create-application.facade.ts b/ui/src/app/features/create-application/create-application.facade.ts new file mode 100644 index 0000000000..f942fd4dbe --- /dev/null +++ b/ui/src/app/features/create-application/create-application.facade.ts @@ -0,0 +1,34 @@ +import { Injectable } from '@angular/core'; +import { Store } from '@ngrx/store'; +import { Observable } from 'rxjs'; +import { Application } from 'src/app/data/interfaces/application'; +import { AppState } from 'src/app/store'; +import { createApplication } from 'src/app/store/application/action'; +import { selectApplications } from 'src/app/store/application/selectors'; +import { createModel } from 'src/app/store/model/action'; + +@Injectable() +export class CreateApplicationFacade { + applications$: Observable; + + constructor(private store: Store) { + this.applications$ = this.store.select(selectApplications); + } + + createApplication(name: string) { + this.store.dispatch(createApplication({ name })); + } + + createModel(name: string, type: string, applicationId: string, isFirstService = false): void { + this.store.dispatch( + createModel({ + model: { + applicationId, + name, + type, + isFirstService + } + }) + ); + } +} diff --git a/ui/src/app/features/create-application/create-application.module.ts b/ui/src/app/features/create-application/create-application.module.ts new file mode 100644 index 0000000000..60fe01c1b1 --- /dev/null +++ b/ui/src/app/features/create-application/create-application.module.ts @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { CommonModule } from '@angular/common'; +import { NgModule } from '@angular/core'; +import { ReactiveFormsModule } from '@angular/forms'; +import { MatButtonModule } from '@angular/material/button'; +import { TranslateModule } from '@ngx-translate/core'; +import { CreateApplicationComponent } from './create-application.component'; +import { MatStepperModule } from '@angular/material/stepper'; +import { MatInputModule } from '@angular/material/input'; +import { MatSelectModule } from '@angular/material/select'; +import { ApplicationNameComponent } from './application/application-name.component'; +import { CreateServiceComponent } from './service/create-service/create-service.component'; +import { ServiceInfoComponent } from './service/service-info/service-info.component'; +import { CreateApplicationFacade } from './create-application.facade'; + +@NgModule({ + declarations: [CreateApplicationComponent, ApplicationNameComponent, CreateServiceComponent, ServiceInfoComponent], + imports: [CommonModule, MatButtonModule, MatStepperModule, MatSelectModule, MatInputModule, TranslateModule, ReactiveFormsModule], + providers: [CreateApplicationFacade], + exports: [CreateApplicationComponent], +}) +export class CreateApplicationModule {} diff --git a/ui/src/app/features/create-application/service/create-service/create-service.component.html b/ui/src/app/features/create-application/service/create-service/create-service.component.html new file mode 100644 index 0000000000..f5b53d5770 --- /dev/null +++ b/ui/src/app/features/create-application/service/create-service/create-service.component.html @@ -0,0 +1,52 @@ + +
+

+ {{ 'create_application_wizard.service.title' | translate }} +

+ +

+ {{ 'create_application_wizard.service.info_about_service' | translate }} +

+ +
+

+ {{ 'create_application_wizard.service.service_name_question' | translate }} +

+ + + +

+ {{ 'create_application_wizard.service.service_type' | translate }} +

+ + + {{ service }} + + +
+ +
+
+
diff --git a/ui/src/app/features/create-application/service/create-service/create-service.component.ts b/ui/src/app/features/create-application/service/create-service/create-service.component.ts new file mode 100644 index 0000000000..d4752176e4 --- /dev/null +++ b/ui/src/app/features/create-application/service/create-service/create-service.component.ts @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { Component, Input, OnDestroy, OnInit } from '@angular/core'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { Subscription } from 'rxjs'; +import { ServiceTypes } from 'src/app/data/enums/service-types.enum'; +import { Application } from 'src/app/data/interfaces/application'; +import { CreateApplicationFacade } from '../../create-application.facade'; + +@Component({ + selector: 'create-service', + templateUrl: './create-service.component.html', + styleUrls: ['../../shared-styles.component.scss'], +}) +export class CreateServiceComponent implements OnInit, OnDestroy { + serviceTypes = [ServiceTypes.Recognition, ServiceTypes.Detection, ServiceTypes.Verification]; + service: FormGroup; + appSubs: Subscription; + currentApplication: Application; + + @Input() applicationName: string; + + constructor(private fb: FormBuilder, private appFacade: CreateApplicationFacade) { } + + ngOnInit() { + this.appSubs = this.appFacade.applications$.subscribe( + result => (this.currentApplication = result.find(app => app.name === this.applicationName)) + ); + + this.service = this.fb.group({ + serviceName: ['', Validators.required], + serviceType: [ServiceTypes.Recognition, Validators.required], + }); + } + + onSave() { + const serviceValue = this.service.value; + const isFirstService = true; + this.appFacade.createModel( + serviceValue.serviceName, + serviceValue.serviceType, + this.currentApplication.id, + isFirstService); + } + + ngOnDestroy() { + this.appSubs.unsubscribe(); + } +} diff --git a/ui/src/app/features/create-application/service/service-info/service-info.component.html b/ui/src/app/features/create-application/service/service-info/service-info.component.html new file mode 100644 index 0000000000..169099f49c --- /dev/null +++ b/ui/src/app/features/create-application/service/service-info/service-info.component.html @@ -0,0 +1,44 @@ + + +
+
+

+ {{ 'create_application_wizard.service.recognition' | translate }} +

+

+ {{ 'create_application_wizard.service.recognition_type' | translate }} +

+
+
+

+ {{ 'create_application_wizard.service.detection' | translate }} +

+

+ {{ 'create_application_wizard.service.detection_type' | translate }} +

+
+ +
+

+ {{ 'create_application_wizard.service.verification' | translate }} +

+

+ {{ 'create_application_wizard.service.verification_type' | translate }} +

+
+ +
\ No newline at end of file diff --git a/ui/src/app/features/create-application/service/service-info/service-info.component.scss b/ui/src/app/features/create-application/service/service-info/service-info.component.scss new file mode 100644 index 0000000000..386ac14670 --- /dev/null +++ b/ui/src/app/features/create-application/service/service-info/service-info.component.scss @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'colors.scss'; +@import 'sizes.scss'; + +.service { + &__type { + margin-bottom: 30px; + &--name { + font-weight: bold; + font-size: $text-md; + margin: 5px 0; + } + + &--info { + font-weight: 400; + font-size: $text-sm; + margin: 5px 0; + } + } +} +p { + color: $white; +} diff --git a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelResponseDto.java b/ui/src/app/features/create-application/service/service-info/service-info.component.ts similarity index 71% rename from java/admin/src/main/java/com/exadel/frs/dto/ui/ModelResponseDto.java rename to ui/src/app/features/create-application/service/service-info/service-info.component.ts index 21c45f8827..ee83ac6541 100644 --- a/java/admin/src/main/java/com/exadel/frs/dto/ui/ModelResponseDto.java +++ b/ui/src/app/features/create-application/service/service-info/service-info.component.ts @@ -1,29 +1,24 @@ -/* - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.exadel.frs.dto.ui; - -import com.exadel.frs.commonservice.enums.ModelType; -import lombok.Data; - -@Data -public class ModelResponseDto { - - private String id; - private String name; - private String apiKey; - private ModelType type; -} \ No newline at end of file +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { Component } from '@angular/core'; + +@Component({ + selector: 'service-info', + templateUrl: './service-info.component.html', + styleUrls: ['./service-info.component.scss'], +}) +export class ServiceInfoComponent {} diff --git a/ui/src/app/features/create-application/shared-styles.component.scss b/ui/src/app/features/create-application/shared-styles.component.scss new file mode 100644 index 0000000000..e63672e24b --- /dev/null +++ b/ui/src/app/features/create-application/shared-styles.component.scss @@ -0,0 +1,61 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'mixins.scss'; +@import 'forms.scss'; +@import 'colors.scss'; +@import 'sizes.scss'; + +.application { + &__title { + font-size: $text-lg; + font-weight: bold; + line-height: 60px; + color: $light-blue; + } + + &__info { + font-size: $text-md; + max-width: 565px; + margin-bottom: 40px; + color: $secondary; + } + + &__question { + font-size: $text-md; + color: $secondary; + } + + &__form { + @include form-field($light-gray, $lighter-gray); + max-width: 500px; + width: 90%; + + .btn { + margin-top: 50px; + button { + @include action-btns(157px, $orange, $white); + border: 2px solid $orange; + + &:disabled { + background: none; + border: 2px solid $gray; + color: $gray; + } + } + } + } +} diff --git a/ui/src/app/features/create-dialog/create-dialog.component.html b/ui/src/app/features/create-dialog/create-dialog.component.html index 8a41340c36..76a5369d47 100644 --- a/ui/src/app/features/create-dialog/create-dialog.component.html +++ b/ui/src/app/features/create-dialog/create-dialog.component.html @@ -14,19 +14,47 @@ ~ permissions and limitations under the License. --> -

{{ 'common.create' | translate }} {{ data.entityType }}

-
- - {{ data.placeholder }} - - -
-
- - + +
+ +
- diff --git a/ui/src/app/features/create-dialog/create-dialog.component.scss b/ui/src/app/features/create-dialog/create-dialog.component.scss new file mode 100644 index 0000000000..8511f9d357 --- /dev/null +++ b/ui/src/app/features/create-dialog/create-dialog.component.scss @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language go verning + * permissions and limitations under the License. + */ + +@import 'forms.scss'; +@import 'mixins.scss'; +@import 'colors.scss'; +@import 'sizes.scss'; + +.form-field { + @include form-field($light-gray, $lighter-gray); +} + +.dialog { + &__header { + width: 100%; + &--title { + @include dialog-title; + } + } + + &__content { + margin-bottom: 40px; + } + + &__actions { + display: flex; + justify-content: space-between; + + .cancel-btn { + @include action-btns(149px, $light-blue, $white); + border: 2px solid $light-blue; + } + .save-btn { + @include action-btns(149px, $white, $light-blue); + } + } +} diff --git a/ui/src/app/features/create-dialog/create-dialog.component.ts b/ui/src/app/features/create-dialog/create-dialog.component.ts index 438920b98c..bc8d1c9848 100644 --- a/ui/src/app/features/create-dialog/create-dialog.component.ts +++ b/ui/src/app/features/create-dialog/create-dialog.component.ts @@ -15,15 +15,20 @@ */ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { MAX_INPUT_LENGTH } from 'src/app/core/constants'; @Component({ selector: 'app-create-dialog', templateUrl: './create-dialog.component.html', + styleUrls: ['./create-dialog.component.scss'], }) export class CreateDialogComponent { + alreadyExists: boolean; + maxInputLength: number = MAX_INPUT_LENGTH; + constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any) {} - onCancelClick(): void { - this.dialogRef.close(); + onChange(name): void { + this.alreadyExists = !!this.data.nameList.find(appName => appName === name); } } diff --git a/ui/src/app/features/delete-dialog/delete-dialog.component.html b/ui/src/app/features/delete-dialog/delete-dialog.component.html index b91d399f15..366782e6a6 100644 --- a/ui/src/app/features/delete-dialog/delete-dialog.component.html +++ b/ui/src/app/features/delete-dialog/delete-dialog.component.html @@ -13,65 +13,32 @@ ~ or implied. See the License for the specific language governing ~ permissions and limitations under the License. --> +
+

{{ 'common.capital.delete' | translate }} {{ data.entityType }}

+
+ +

{{ 'users.confirmation_dialog.delete_confirmation_question' | translate }}

+ -

{{ 'org_users.delete_dialog.title' | translate: { type: data.entityType } }}

-
- - -

- {{ - 'app_users.delete_dialog.confirmation_question' - | translate: { name: data.entityName, application: data.organizationName || data.applicationName } - }} + +

+ {{ 'users.confirmation_dialog.warning_msg_owner' | translate: { user: data.user, applicationNames: appNames } }}

- - -

- {{ - 'org_users.delete_dialog.warn_information' | translate: { firstName: data.entity.firstName, lastName: data.entity.lastName } - }} - {{ data.entity.ownerOfApplications.join(', ') }}. -

-

- {{ 'org_users.delete_dialog.warn_hint_anyway' | translate }} - - {{ 'org_users.delete_dialog.warn_hint_replace_to_owner' | translate }} - - - - {{ 'org_users.delete_dialog.warn_hint_owner' | translate }} - - - - {{ 'org_users.delete_dialog.warn_hint_new_owner' | translate }} - - - {{ option.name }} - - - -

- - -

- {{ - 'org_users.delete_dialog.confirmation_question' | translate: { name: data.entity.firstName, lastName: data.entity.lastName } - }} -

- + +

{{ 'users.confirmation_dialog.warning_msg_admin' | translate: { user: data.user, applicationNames: appNames } }}

 - - -

{{ 'org_users.delete_dialog.default_confirmation' | translate: { type: data.entityType, name: data.entityName } }}

- - -
-
- - + +
+
+ + +
+
- diff --git a/ui/src/app/features/delete-dialog/delete-dialog.component.scss b/ui/src/app/features/delete-dialog/delete-dialog.component.scss new file mode 100644 index 0000000000..ee3d19ca53 --- /dev/null +++ b/ui/src/app/features/delete-dialog/delete-dialog.component.scss @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +@import 'mixins.scss'; +@import 'colors.scss'; + +.delete-container { + position: relative; + + .mat-dialog-title { + @include dialog-title; + margin-bottom: 40px; + } + + .description { + color: $md-blue; + p { + margin-bottom: 40px; + } + } + + .actions { + display: flex; + justify-content: space-between; + + .delete-btn { + @include action-btns(170px, $white, $light-blue); + } + + .cancel-btn { + @include action-btns(149px, $light-blue, $white); + border: 2px solid $light-blue; + } + } + + .button-cross { + position: absolute; + top: -1rem; + right: -0.6rem; + + .mat-icon { + @include sm-icon-size(16px); + } + } +} diff --git a/ui/src/app/features/delete-dialog/delete-dialog.component.ts b/ui/src/app/features/delete-dialog/delete-dialog.component.ts index d47d7a8ce1..912ab5e04b 100644 --- a/ui/src/app/features/delete-dialog/delete-dialog.component.ts +++ b/ui/src/app/features/delete-dialog/delete-dialog.component.ts @@ -13,13 +13,22 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { Component, Inject } from '@angular/core'; +import { Component, Inject, OnInit } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; @Component({ selector: 'app-delete-dialog', templateUrl: './delete-dialog.component.html', + styleUrls: ['./delete-dialog.component.scss'], }) -export class DeleteDialogComponent { +export class DeleteDialogComponent implements OnInit { + appNames: string; + constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any) {} + + ngOnInit() { + if (!!this.data.appList?.length) { + this.appNames = this.data.appList.map(app => ' ' + app.name).toString(); + } + } } diff --git a/ui/src/app/features/drag-n-drop/drag-n-drop.component.html b/ui/src/app/features/drag-n-drop/drag-n-drop.component.html index 622aac92e6..bba6c1fed2 100644 --- a/ui/src/app/features/drag-n-drop/drag-n-drop.component.html +++ b/ui/src/app/features/drag-n-drop/drag-n-drop.component.html @@ -15,14 +15,12 @@ -->
-
-
- -
+
-
{{ title }}
-
{{ label }}
+

{{ title }}

+

{{ label }} {{ maxImageSize }}

- + +
diff --git a/ui/src/app/features/drag-n-drop/drag-n-drop.component.scss b/ui/src/app/features/drag-n-drop/drag-n-drop.component.scss index ff6b4c1365..c80a9a4227 100644 --- a/ui/src/app/features/drag-n-drop/drag-n-drop.component.scss +++ b/ui/src/app/features/drag-n-drop/drag-n-drop.component.scss @@ -14,34 +14,16 @@ * permissions and limitations under the License. */ -@import "colors.scss"; +@import 'colors.scss'; +@import 'sizes.scss'; $border-radius: 12px; -@mixin content($max-width, $margin-title) { - .content { - max-width: $max-width; - - .title { - margin-bottom: $margin-title; - font-size: 22px; - line-height: 125%; - font-weight: 600; - } - - .label { - color: $light-gray; - white-space: pre-line; - font-size: 16px; - line-height: 140%; - } - } -} - .dropzone { position: relative; - padding: 24px; - background-color: $white; + background: $lighter-blue; + opacity: 0.8; + border-radius: $border-radius; &--container { @@ -49,56 +31,58 @@ $border-radius: 12px; justify-content: center; align-items: center; width: 100%; - border: dashed 2px $light-gray; + height: 135px; + font-weight: 800; + font-size: $text-sm; + border: dashed 2px $dark-gray; border-radius: $border-radius; - transition: .2s; - - &.column { - padding: 55px 0 24px; - flex-direction: column; - text-align: center; + transition: 0.2s; - @include content(400px, 16px); - - .icon { - margin-bottom: 48px; - } + &.testing { + height: 250px; } + } - &.row { - padding: 16px 0; - flex-direction: row; - text-align: left; - - @include content(556px, 8px); - - .icon { - margin-right: 21px; - } + .title { + text-align: center; + font-size: $text-md; + color: $dark-blue; + margin: 0; + + &.testing { + margin: auto; + max-width: 450px; + line-height: 35px; + font-size: $text-lg; + color: $dark-blue; } + } - .icon { - width: 53px; - height: 48px; - } + .description { + text-align: center; + font-weight: 400; + font-size: $text-md; + margin: 1rem auto 0; + max-width: 430px; } input { - opacity: 0; - position: absolute; - z-index: 2; width: 100%; height: 100%; + cursor: pointer; + position: absolute; + z-index: 2; top: 0; left: 0; - cursor: pointer; + opacity: 0; } - &:hover &--container { - transition: .2s; - background-color: $special-light-blue; - border-color: $special-blue; + &:hover { + transition: 0.2s; + opacity: 1; } -} - + &_inline { + padding: 0; + } +} diff --git a/ui/src/app/features/drag-n-drop/drag-n-drop.component.ts b/ui/src/app/features/drag-n-drop/drag-n-drop.component.ts index 9df810c9f0..a9850c6d08 100644 --- a/ui/src/app/features/drag-n-drop/drag-n-drop.component.ts +++ b/ui/src/app/features/drag-n-drop/drag-n-drop.component.ts @@ -13,36 +13,45 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -import { AfterViewInit, Component, ElementRef, EventEmitter, Input, Output, Renderer2, ViewChild } from '@angular/core'; +import { Component, ElementRef, EventEmitter, Input, Output, ViewChild } from '@angular/core'; +import { ActivatedRoute, Router } from '@angular/router'; +import { ServiceTypes } from 'src/app/data/enums/service-types.enum'; @Component({ selector: 'app-drag-n-drop', templateUrl: './drag-n-drop.component.html', styleUrls: ['./drag-n-drop.component.scss'], }) -export class DragNDropComponent implements AfterViewInit { +export class DragNDropComponent { @ViewChild('fileDropRef') fileDropEl: ElementRef; @Input() title: string; @Input() label: string; - @Input() model: any; - @Input('viewComponentColumn') - get view() { - return this.viewColumn; - } - set view(val: boolean) { - this.viewColumn = true; + @Input() maxImageSize: string; + + displayDescription: boolean; + + constructor(private router: Router) {} + + ngOnInit(): void { + this.router.url.includes('manage-collection') ? (this.displayDescription = false) : (this.displayDescription = true); } - @Output() upload: EventEmitter = new EventEmitter(); + serviceType: ServiceTypes; - viewColumn = false; + viewComponentColumn: boolean; + @Input('viewComponentColumn') set setViewComponentColumn(val: boolean | '') { + this.viewComponentColumn = val === '' || val; + } - constructor(private renderer: Renderer2, private elementRef: ElementRef) {} + inline: boolean; + @Input('inline') set setInline(val: boolean | '') { + this.inline = val === '' || val; + } - ngAfterViewInit(): void { - const nativeElement: ChildNode = this.elementRef.nativeElement.firstChild.firstChild; - const classValue = this.viewColumn ? 'column' : 'row'; + @Output() upload: EventEmitter = new EventEmitter(); - this.renderer.addClass(nativeElement, classValue); + onChange(event): void { + this.fileBrowseHandler(event.target.files); + this.fileDropEl.nativeElement.value = null; } /** @@ -65,9 +74,9 @@ export class DragNDropComponent implements AfterViewInit { * @param files (Files List) * TODO Send file to api */ - uploadFile(files: Array) { + uploadFile(files: FileList) { if (files.length > 0) { - this.upload.emit(files[0]); + this.upload.emit(Array.from(files)); } } } diff --git a/ui/src/app/features/edit-dialog/edit-dialog.component.html b/ui/src/app/features/edit-dialog/edit-dialog.component.html index 7cd87a0a3d..ca347ed900 100644 --- a/ui/src/app/features/edit-dialog/edit-dialog.component.html +++ b/ui/src/app/features/edit-dialog/edit-dialog.component.html @@ -14,19 +14,73 @@ ~ permissions and limitations under the License. --> -

{{ 'common.capital.edit' | translate }} {{ data.entityType }}

-
- - {{ 'common.name' | translate }} - - -
-
- - + + +
+ + +
+ + + + + {{ 'common.capital.delete' | translate }} {{ data.type }} + + +
+ + {{ 'models.delete_dialog.confirmation_question' | translate: { entityType: data.type | lowercase } }} + +
+ +
+ +
+ + +
+ +
- diff --git a/ui/src/app/features/edit-dialog/edit-dialog.component.scss b/ui/src/app/features/edit-dialog/edit-dialog.component.scss new file mode 100644 index 0000000000..1d8a66bdef --- /dev/null +++ b/ui/src/app/features/edit-dialog/edit-dialog.component.scss @@ -0,0 +1,117 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +@import 'forms.scss'; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'sizes.scss'; + +.form-field { + @include form-field($light-gray, $lighter-gray); +} + +.container { + width: 100%; + + &--title { + font-size: $text-lg; + font-weight: 600; + color: $dark-blue; + } + + .content { + margin-bottom: 30px; + + .label, + .type { + color: $dark-blue; + } + + .label { + font-weight: bold; + font-size: $text-md; + } + .type { + font-size: $text-sm; + text-transform: capitalize; + } + .owner { + font-size: $text-sm; + } + } + .action-btns { + width: 100%; + display: flex; + justify-content: space-between; + + .cancel-btn { + @include action-btns(129px, $light-blue, $white); + border: 2px solid $light-blue; + } + + .save-btn { + @include action-btns(140px, $white, $light-blue); + } + } + + .delete-container { + display: flex; + flex-direction: column; + align-items: flex-start; + justify-content: space-between; + min-height: 100px; + + .confirmation--question { + font-size: $text-sm; + color: $secondary; + } + + .del-btn { + @include action-btns(200px, $white, $light-blue); + font-size: $text-sm; + height: 37px; + border-radius: 10px; + margin-top: 20px !important; + } + + .mat-expansion-panel { + background: none; + color: $secondary; + } + + .mat-expansion-panel-header { + padding: 0; + margin-bottom: 20px; + cursor: pointer; + display: flex; + align-items: flex-end; + font-weight: bold; + border: none; + background: none; + font-size: $text-md; + + .mat-icon { + display: flex; + align-items: center; + margin-right: 15px; + svg { + width: 27px; + height: 27px; + font-size: 27px; + } + } + } + } +} diff --git a/ui/src/app/features/edit-dialog/edit-dialog.component.ts b/ui/src/app/features/edit-dialog/edit-dialog.component.ts index 199b836019..1b352bff4b 100644 --- a/ui/src/app/features/edit-dialog/edit-dialog.component.ts +++ b/ui/src/app/features/edit-dialog/edit-dialog.component.ts @@ -15,16 +15,22 @@ */ import { ChangeDetectionStrategy } from '@angular/core'; -import { Component, Inject, OnInit } from '@angular/core'; +import { Component, Inject } from '@angular/core'; import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { MAX_INPUT_LENGTH } from 'src/app/core/constants'; @Component({ selector: 'app-edit-dialog', templateUrl: './edit-dialog.component.html', + styleUrls: ['./edit-dialog.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) export class EditDialogComponent { initialName: string; + panelOpenState: boolean = false; + deleteInput: string = ''; + alreadyExists: boolean; + maxInputLength: number = MAX_INPUT_LENGTH; get isRenameDisabled(): any { return this.data.entityName === this.initialName || !this.data.entityName; @@ -34,7 +40,18 @@ export class EditDialogComponent { this.initialName = data.entityName; } - onCancelClick(): void { - this.dialogRef.close(); + onChange(name): void { + this.alreadyExists = !!this.data.models.find(model => model.name === name); + } + + onSave() { + this.dialogRef.close({ + update: true, + name: this.initialName, + }); + } + + onDelete() { + this.dialogRef.close({ update: false }); } } diff --git a/ui/src/app/features/edit-subject/edit-subject-dialog.component.html b/ui/src/app/features/edit-subject/edit-subject-dialog.component.html new file mode 100644 index 0000000000..2d16c96d0b --- /dev/null +++ b/ui/src/app/features/edit-subject/edit-subject-dialog.component.html @@ -0,0 +1,50 @@ + + +
+

{{ 'common.capital.edit' | translate }} {{ data.type }}

+
+
+ + +
+ {{ 'common.input_lengh_err' | translate }} +
+
+ +
+ + +
+
+ +
diff --git a/ui/src/app/features/user-table/user-table.component.scss b/ui/src/app/features/edit-subject/edit-subject-dialog.component.scss similarity index 50% rename from ui/src/app/features/user-table/user-table.component.scss rename to ui/src/app/features/edit-subject/edit-subject-dialog.component.scss index 2ff3fe319b..cc995020fd 100644 --- a/ui/src/app/features/user-table/user-table.component.scss +++ b/ui/src/app/features/edit-subject/edit-subject-dialog.component.scss @@ -1,66 +1,53 @@ -/*! - * Copyright (c) 2020 the original author or authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ -@import "colors.scss"; -@import "mixins.scss"; - -caption { - display: none; -} - -.mat { - - &-header-row { - display: none; - } - - &-column { - - &-user { - width: 100%; - - .user { - - &--name { - @include not-break-string(); - } - - &--role { - color: $light-gray; - } - } - } - - &-role { - width: 44px; - text-align: center; - - .mat-icon { - vertical-align: sub; - @include md-icon-size(); - } - } - - &-delete { - width: 44px; - text-align: center; - - .mat-icon { - vertical-align: sub; - @include md-icon-size(); - } - } - } -} +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'forms.scss'; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'sizes.scss'; + +.form-field { + @include form-field($light-gray, $lighter-gray); +} +.container { + width: 100%; + padding: 0; + + &--title { + font-size: $text-lg; + color: $dark-blue; + margin: 0; + } + + label { + font-size: $text-md; + } + + .actions { + margin-top: 40px; + display: flex; + justify-content: space-between; + + .cancel-btn { + @include action-btns(129px, $light-blue, $white); + border: 2px solid $light-blue; + } + + .save-btn { + @include action-btns(140px, $white, $light-blue); + } + } +} diff --git a/ui/src/app/features/edit-subject/edit-subject-dialog.component.ts b/ui/src/app/features/edit-subject/edit-subject-dialog.component.ts new file mode 100644 index 0000000000..521ef4914b --- /dev/null +++ b/ui/src/app/features/edit-subject/edit-subject-dialog.component.ts @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { Component, Inject } from '@angular/core'; +import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { MAX_INPUT_LENGTH } from 'src/app/core/constants'; + +@Component({ + selector: 'edit-subject', + templateUrl: './edit-subject-dialog.component.html', + styleUrls: ['./edit-subject-dialog.component.scss'], +}) +export class EditSubjectDialog { + initialName: string; + maxInputLength: number = MAX_INPUT_LENGTH; + + get isRenameDisabled(): any { + return this.data.entityName === this.initialName || !this.data.entityName; + } + + constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any) { + this.initialName = data.entityName; + } + + onSave() { + this.dialogRef.close(this.initialName); + } +} diff --git a/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.html b/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.html index 66d7c02d5d..faa2675e91 100644 --- a/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.html +++ b/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.html @@ -14,32 +14,44 @@ ~ permissions and limitations under the License. --> -

{{ 'toolbar.user_info' | translate }}

+

{{ 'toolbar.user_info_change' | translate }}

- - {{ 'registration.first_name' | translate }} - - {{ 'registration.name_restriction' | translate }} - - - +
+ +
+ {{ 'registration.name_restriction' | translate }} +
+
+ {{ 'common.input_lengh_err' | translate }} +
+ +
- - {{ 'registration.last_name' | translate }} - - {{ 'registration.last_name_restriction' | translate }} - - - +
+ +
+ {{ 'registration.last_name_restriction' | translate }} +
+
+ {{ 'common.input_lengh_err' | translate }} +
+ +
+
+ + +
-
- - -
diff --git a/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.scss b/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.scss new file mode 100644 index 0000000000..18234fb608 --- /dev/null +++ b/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.scss @@ -0,0 +1,29 @@ +@import 'forms.scss'; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'sizes.scss'; + +.form-field { + @include form-field($light-gray, $lighter-gray); +} + +.title { + @include dialog-title; +} + +.actions { + margin: 2rem 0 1rem; + width: 100%; + display: flex; + justify-content: space-between; + + .cancel { + @include action-btns(129px, $light-blue, $white); + border: 2px solid $light-blue; + } + + .rename { + @include action-btns(140px, $white, $light-blue); + border: none; + } +} diff --git a/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.ts b/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.ts index fb6baff401..e5fa3afe7c 100644 --- a/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.ts +++ b/ui/src/app/features/edit-user-info-dialog/edit-user-info-dialog.component.ts @@ -17,27 +17,31 @@ import { Component, Inject, OnInit } from '@angular/core'; import { FormBuilder, FormControl, FormGroup, Validators } from '@angular/forms'; import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { MAX_INPUT_LENGTH } from 'src/app/core/constants'; @Component({ selector: 'app-edit-user-info-dialog', templateUrl: './edit-user-info-dialog.component.html', + styleUrls: ['./edit-user-info-dialog.component.scss'], }) export class EditUserInfoDialogComponent implements OnInit { form: FormGroup; firstName: string; lastName: string; + maxLength = MAX_INPUT_LENGTH; constructor( public dialogRef: MatDialogRef, private formBuilder: FormBuilder, - @Inject(MAT_DIALOG_DATA) public data: { userName: string } + @Inject(MAT_DIALOG_DATA) public data: { firstName: string; lastName: string } ) {} ngOnInit(): void { - [this.firstName, this.lastName] = this.data?.userName.split(' '); + this.firstName = this.data?.firstName; + this.lastName = this.data?.lastName; this.form = this.formBuilder.group({ - firstName: new FormControl(this.firstName, [Validators.required]), - lastName: new FormControl(this.lastName, [Validators.required]), + firstName: new FormControl(this.firstName, [Validators.required, Validators.maxLength(this.maxLength)]), + lastName: new FormControl(this.lastName, [Validators.required, Validators.maxLength(this.maxLength)]), }); } diff --git a/ui/src/app/features/face-services/face-picture/face-picture.component.html b/ui/src/app/features/face-services/face-picture/face-picture.component.html index 7da2eab1e9..a74786e61a 100644 --- a/ui/src/app/features/face-services/face-picture/face-picture.component.html +++ b/ui/src/app/features/face-services/face-picture/face-picture.component.html @@ -21,6 +21,7 @@ + @@ -31,7 +32,7 @@
{{ value.subject }}
-
Similarity: {{ value.similarity | number: '1.0-2' }}
+
Similarity: {{ value.similarity | number : '1.0-2' }}
 @@ -40,11 +41,11 @@ -
Probability: {{ face.box.probability | number: '1.0-2' }}
+
Probability: {{ face.box.probability | number : '1.0-2' }}
 -
Similarity: {{ face.similarity | number: '1.0-2' }}
+
Similarity: {{ face.similarity | number : '1.0-2' }}
{{ face.gender.value | titlecase }}, Age: {{ face.age.low + '-' + face.age.high }}
diff --git a/ui/src/app/features/face-services/face-picture/face-picture.component.scss b/ui/src/app/features/face-services/face-picture/face-picture.component.scss index 96dd1e614b..c6bbd9a990 100644 --- a/ui/src/app/features/face-services/face-picture/face-picture.component.scss +++ b/ui/src/app/features/face-services/face-picture/face-picture.component.scss @@ -13,8 +13,9 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -@import "colors.scss"; -@import "mixins.scss"; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'sizes.scss'; .face-displaying { display: flex; @@ -43,6 +44,11 @@ left: 0; } + .canvas-pose { + position: absolute; + left: 0; + } + .face-box { position: absolute; display: flex; @@ -65,7 +71,7 @@ position: relative; margin-top: 18px; padding: 8px 20px; - max-width: 170px; + max-width: 200px; width: fit-content; background-color: $light-blue; border-radius: 6px; @@ -90,20 +96,19 @@ @include not-break-string; margin-bottom: 4px; font-weight: 600; - font-size: 16px; + font-size: $text-md; } &_similarity { @include not-break-string; margin-bottom: 6px; font-weight: normal; - font-size: 14px; + font-size: $text-sm; } &_gender { @include not-break-string; - font-weight: 500; - font-size: 14px; + font-size: $text-sm; } } } diff --git a/ui/src/app/features/face-services/face-picture/face-picture.component.ts b/ui/src/app/features/face-services/face-picture/face-picture.component.ts index 0d1dd09013..794f5533e0 100644 --- a/ui/src/app/features/face-services/face-picture/face-picture.component.ts +++ b/ui/src/app/features/face-services/face-picture/face-picture.component.ts @@ -35,12 +35,17 @@ export class FacePictureComponent implements OnChanges, OnInit { @Input() set showLandmarks(value: boolean) { this.disableLandmarks = value; } + @Input() set showPose(value: boolean) { + this.disablePose = value; + } @ViewChild('canvasPicture', { static: true }) canvasPicture: ElementRef; @ViewChild('canvasLandmarks', { static: true }) canvasLandmarks: ElementRef; + @ViewChild('canvasPose', { static: true }) canvasPose: ElementRef; types = ServiceTypes; disableLandmarks = false; + disablePose = false; private currentPicture: () => any; @@ -53,6 +58,7 @@ export class FacePictureComponent implements OnChanges, OnInit { this.initCanvasSize(); if (!!this.currentPicture) this.currentPicture = this.loadPicture(this.canvasPicture, this.picture, this.currentPicture); this.getLandmarks(this.canvasLandmarks, this.printData); + this.getPose(this.canvasPose, this.printData); } initCanvasSize(): void { @@ -61,6 +67,9 @@ export class FacePictureComponent implements OnChanges, OnInit { this.canvasLandmarks.nativeElement.setAttribute('height', String(this.picture.sizeCanvas.height)); this.canvasLandmarks.nativeElement.setAttribute('width', String(this.picture.sizeCanvas.width)); + + this.canvasPose.nativeElement.setAttribute('height', String(this.picture.sizeCanvas.height)); + this.canvasPose.nativeElement.setAttribute('width', String(this.picture.sizeCanvas.width)); } loadPicture(canvasEl: ElementRef, picture, currentPicture): () => any { @@ -87,7 +96,7 @@ export class FacePictureComponent implements OnChanges, OnInit { // eslint-disable-next-line @typescript-eslint/naming-convention const { x_max, x_min, y_max, y_min } = data.box; const frameArea: number = (Math.round(x_max - x_min) * Math.round(y_max - y_min)) / 19000; - const sizePoint = frameArea > 1.4 || data.landmarks.length >= 108 ? frameArea : 1.4; + const sizePoint = frameArea > 1.4 || data.landmarks.length >= 108 ? frameArea / 2 : 1.4; data.landmarks.forEach(landmark => { ctx.beginPath(); @@ -98,4 +107,64 @@ export class FacePictureComponent implements OnChanges, OnInit { ctx.stroke(); }); } + + getPose(canvasEl: ElementRef, data: RequestResultRecognition[] | FaceMatches[] | SourceImageFace[]): void { + if (!data) return; + + const ctx: CanvasRenderingContext2D = canvasEl.nativeElement.getContext('2d'); + + data.forEach(val => this.displayPose(ctx, val)); + } + + displayPose(ctx: CanvasRenderingContext2D, data: RequestResultRecognition | FaceMatches | SourceImageFace): void { + const { x_max, x_min, y_max, y_min } = data.box; + + //tdx, tdy starting points for drawing. + const tdx = (x_max + x_min) / 2; + const tdy = (y_max + y_min) / 2; + + // size is size of axis. + const size = 70; + + const pitch = data.pose.pitch * Math.PI / 180; + const yaw = -(data.pose.yaw * Math.PI / 180); + const roll = data.pose.roll * Math.PI / 180; + + const xAxisX = size * (Math.cos(yaw) * Math.cos(roll)) + tdx; + const xAxisY = size * (Math.cos(pitch) * Math.sin(roll) + Math.cos(roll) * Math.sin(pitch) * Math.sin(yaw)) + tdy + + const yAxisX = size * (-Math.cos(yaw) * Math.sin(roll)) + tdx + const yAxisY = size * (Math.cos(pitch) * Math.cos(roll) - Math.sin(pitch) * Math.sin(yaw) * Math.sin(roll)) + tdy + + const zAxisX = size * (Math.sin(yaw)) + tdx + const zAxisY = size * (-Math.cos(yaw) * Math.sin(pitch)) + tdy + + const graphLines = [ + { + color: "#FF0000", + x: xAxisX, + y: xAxisY + }, + { + color: "#00FF00", + x: yAxisX, + y: yAxisY + }, + { + color: "#0000FF", + x: zAxisX, + y: zAxisY + } + ]; + + graphLines.forEach(val => { + ctx.beginPath(); + ctx.moveTo(tdx, tdy); + ctx.strokeStyle = val.color; + ctx.lineTo(val.x, val.y); + ctx.lineWidth = 5; + ctx.stroke(); + ctx.closePath(); + }) + } } diff --git a/ui/src/app/features/face-services/face-recognition/face-recognition-container.component.html b/ui/src/app/features/face-services/face-recognition/face-recognition-container.component.html index 7976cb1dc6..8f8235931c 100644 --- a/ui/src/app/features/face-services/face-recognition/face-recognition-container.component.html +++ b/ui/src/app/features/face-services/face-recognition/face-recognition-container.component.html @@ -20,6 +20,7 @@ [requestInfo]="requestInfo$ | async" [printData]="data$ | async" [type]="type" + [maxImageSize]="maxImageSize" (selectFile)="recognizeFace($event)" (resetFace)="resetFace()" > diff --git a/ui/src/app/features/face-services/face-recognition/face-recognition-container.component.ts b/ui/src/app/features/face-services/face-recognition/face-recognition-container.component.ts index 731e2dca10..e3151150be 100644 --- a/ui/src/app/features/face-services/face-recognition/face-recognition-container.component.ts +++ b/ui/src/app/features/face-services/face-recognition/face-recognition-container.component.ts @@ -17,9 +17,9 @@ import { ChangeDetectionStrategy, Component, Input, OnDestroy, OnInit } from '@a import { Store } from '@ngrx/store'; import { switchMap } from 'rxjs/operators'; -import { defer, Observable, of } from 'rxjs'; +import { defer, Observable, of, Subscription } from 'rxjs'; -import { AVAILABLE_IMAGE_EXTENSIONS, MAX_IMAGE_SIZE } from 'src/app/core/constants'; +import { AVAILABLE_IMAGE_EXTENSIONS } from 'src/app/core/constants'; import { AppState } from '../../../store'; import { recognizeFace, recognizeFaceReset } from '../../../store/face-recognition/action'; @@ -34,6 +34,7 @@ import { getFileExtension } from '../face-services.helpers'; import { SnackBarService } from '../../snackbar/snackbar.service'; import { ServiceTypes } from '../../../data/enums/service-types.enum'; import { LoadingPhotoService } from '../../../core/photo-loader/photo-loader.service'; +import { selectMaxFileSize } from 'src/app/store/image-size/selectors'; @Component({ selector: 'app-face-recognition-container', @@ -47,6 +48,8 @@ export class FaceRecognitionContainerComponent implements OnInit, OnDestroy { requestInfo$: Observable; pending$: Observable; isLoaded$: Observable; + maxImageSize: number; + imageSizeSubs: Subscription; @Input() title: string; @Input() type: ServiceTypes; @@ -61,6 +64,7 @@ export class FaceRecognitionContainerComponent implements OnInit, OnDestroy { this.photo$ = this.store .select(selectFile) .pipe(switchMap(file => defer(() => (!!file ? this.loadingPhotoService.loader(file) : of(null))))); + this.imageSizeSubs = this.store.select(selectMaxFileSize).subscribe(res => (this.maxImageSize = res.clientMaxFileSize)); } resetFace(): void { @@ -74,7 +78,7 @@ export class FaceRecognitionContainerComponent implements OnInit, OnDestroy { messageOptions: { filename: file.name }, type: 'error', }); - } else if (file.size > MAX_IMAGE_SIZE) { + } else if (file.size > this.maxImageSize) { this.snackBarService.openNotification({ messageText: 'face_recognition_container.file_size_error', type: 'error' }); } else { this.store.dispatch(recognizeFace({ file })); @@ -83,5 +87,6 @@ export class FaceRecognitionContainerComponent implements OnInit, OnDestroy { ngOnDestroy() { this.resetFace(); + this.imageSizeSubs.unsubscribe(); } } diff --git a/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.html b/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.html index bdbab7fc91..aee8d6e878 100644 --- a/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.html +++ b/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.html @@ -17,9 +17,10 @@
@@ -29,9 +30,11 @@ #changePhoto [disabledButtons]="pending" [disabledLandmarksButton]="!recalculatePrintData" + [disabledPoseButton]="!recalculatePrintData" (resetPhoto)="onResetFile()" (changePhoto)="onResetFile($event)" (addLandmark)="changePhoto.showLandmarks = !changePhoto.showLandmarks" + (addPose)="changePhoto.showPose = !changePhoto.showPose" > diff --git a/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.scss b/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.scss index ad84f3a499..de71399f5c 100644 --- a/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.scss +++ b/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.scss @@ -13,8 +13,8 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -@import "media.scss"; -@import "mixins.scss"; +@import 'media.scss'; +@import 'mixins.scss'; .result { margin-top: 24px; @@ -22,16 +22,16 @@ grid-template-columns: 1fr 1fr; width: 100%; grid-gap: 20px; - grid-template-areas:"drag-n-drop drag-n-drop" "process-img process-img" "request response"; + grid-template-areas: 'drag-n-drop drag-n-drop' 'process-img process-img' 'request response'; @include mobile-portrait { grid-template-columns: 1fr; - grid-template-areas:"process-img" "request" "response"; + grid-template-areas: 'process-img' 'request' 'response'; } @include mobile { grid-template-columns: 1fr; - grid-template-areas:"process-img" "request" "response"; + grid-template-areas: 'process-img' 'request' 'response'; } .drag-n-drop { @@ -53,3 +53,7 @@ min-width: 100%; } } + +.mat-expansion-panel-header-title { + font-size: $text-md; +} diff --git a/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.ts b/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.ts index e23f8fc7a3..c5bbe84754 100644 --- a/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.ts +++ b/ui/src/app/features/face-services/face-recognition/recognition-result/recognition-result.component.ts @@ -40,6 +40,7 @@ export class RecognitionResultComponent implements OnChanges { @Input() isLoaded: boolean; @Input() pending: boolean; @Input() type: ServiceTypes; + @Input() maxImageSize: number; @Output() selectFile = new EventEmitter(); @Output() resetFace = new EventEmitter(); diff --git a/ui/src/app/features/face-services/face-verification/face-verification-container.component.html b/ui/src/app/features/face-services/face-verification/face-verification-container.component.html index 9a0a21e51d..d5fc99cee8 100644 --- a/ui/src/app/features/face-services/face-verification/face-verification-container.component.html +++ b/ui/src/app/features/face-services/face-verification/face-verification-container.component.html @@ -23,6 +23,7 @@ [isLoaded]="isLoaded$ | async" [requestInfo]="requestInfo$ | async" [type]="type" + [maxImageSize]="maxImageSize" (selectProcessFile)="processFileUpload($event)" (selectCheckFile)="checkFileUpload($event)" (resetProcessFile)="processFileReset($event)" diff --git a/ui/src/app/features/face-services/face-verification/face-verification-container.component.ts b/ui/src/app/features/face-services/face-verification/face-verification-container.component.ts index c98a5763c4..ff641befcc 100644 --- a/ui/src/app/features/face-services/face-verification/face-verification-container.component.ts +++ b/ui/src/app/features/face-services/face-verification/face-verification-container.component.ts @@ -16,10 +16,10 @@ import { ChangeDetectionStrategy, Component, Input, OnDestroy, OnInit } from '@angular/core'; import { Store } from '@ngrx/store'; -import { defer, Observable, of } from 'rxjs'; +import { defer, Observable, of, Subscription } from 'rxjs'; import { map, switchMap } from 'rxjs/operators'; -import { AVAILABLE_IMAGE_EXTENSIONS, MAX_IMAGE_SIZE } from 'src/app/core/constants'; +import { AVAILABLE_IMAGE_EXTENSIONS } from 'src/app/core/constants'; import { AppState } from '../../../store'; import { @@ -43,6 +43,7 @@ import { SnackBarService } from '../../snackbar/snackbar.service'; import { ServiceTypes } from '../../../data/enums/service-types.enum'; import { LoadingPhotoService } from '../../../core/photo-loader/photo-loader.service'; import { VerificationServiceFields } from '../../../data/enums/verification-service.enum'; +import { selectMaxFileSize } from 'src/app/store/image-size/selectors'; @Component({ selector: 'app-face-verification-container', @@ -58,6 +59,8 @@ export class FaceVerificationContainerComponent implements OnInit, OnDestroy { requestInfo$: Observable; pending$: Observable; isLoaded$: Observable; + imageSizeSubs: Subscription; + maxImageSize: number; @Input() type: ServiceTypes; @@ -77,10 +80,12 @@ export class FaceVerificationContainerComponent implements OnInit, OnDestroy { this.checkPhoto$ = this.store .select(selectCheckFile) .pipe(switchMap(file => defer(() => (!!file ? this.loadingPhotoService.loader(file) : of(null))))); + this.imageSizeSubs = this.store.select(selectMaxFileSize).subscribe(res => (this.maxImageSize = res.clientMaxFileSize)); } ngOnDestroy() { this.store.dispatch(verifyFaceReset()); + this.imageSizeSubs.unsubscribe(); } processFileUpload(file) { @@ -117,7 +122,7 @@ export class FaceVerificationContainerComponent implements OnInit, OnDestroy { type: 'error', }); return false; - } else if (file.size > MAX_IMAGE_SIZE) { + } else if (file.size > this.maxImageSize) { this.snackBarService.openNotification({ messageText: 'face_recognition_container.file_size_error', type: 'error' }); return false; } else { diff --git a/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.html b/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.html index 5fab1f5323..4bfa9a5a97 100644 --- a/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.html +++ b/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.html @@ -17,9 +17,10 @@
@@ -28,9 +29,11 @@ #changePhoto [disabledButtons]="pending" [disabledLandmarksButton]="!printDataProcessPhoto" + [disabledPoseButton]="!printDataProcessPhoto" (resetPhoto)="onResetProcessFile()" (changePhoto)="onResetProcessFile($event)" (addLandmark)="changePhoto.showLandmarks = !changePhoto.showLandmarks" + (addPose)="changePhoto.showPose = !changePhoto.showPose" > @@ -39,6 +42,7 @@ [printData]="printDataProcessPhoto" [isLoaded]="!pending" [showLandmarks]="changePhoto.showLandmarks" + [showPose]="changePhoto.showPose" > @@ -47,9 +51,10 @@
@@ -58,9 +63,11 @@ #changePhoto [disabledButtons]="pending" [disabledLandmarksButton]="!printDataCheckPhoto" + [disabledPoseButton]="!printDataCheckPhoto" (resetPhoto)="onResetCheckFile()" (changePhoto)="onResetCheckFile($event)" (addLandmark)="changePhoto.showLandmarks = !changePhoto.showLandmarks" + (addPose)="changePhoto.showPose = !changePhoto.showPose" > @@ -70,6 +77,7 @@ [type]="type" [isLoaded]="!pending" [showLandmarks]="changePhoto.showLandmarks" + [showPose]="changePhoto.showPose" > diff --git a/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.scss b/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.scss index 7f86395e95..9be341b93e 100644 --- a/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.scss +++ b/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.scss @@ -13,23 +13,24 @@ * or implied. See the License for the specific language governing * permissions and limitations under the License. */ -@import "media.scss"; +@import 'media.scss'; +@import 'sizes.scss'; .result { display: grid; grid-template-columns: 1fr 1fr; width: 100%; grid-gap: 20px; - grid-template-areas:"process-img check-img" "request response"; + grid-template-areas: 'process-img check-img' 'request response'; @include mobile-portrait { grid-template-columns: 1fr; - grid-template-areas:"process-img" " check-img" "request" "response"; + grid-template-areas: 'process-img' ' check-img' 'request' 'response'; } @include mobile { grid-template-columns: 1fr; - grid-template-areas:"process-img" " check-img" "request" "response"; + grid-template-areas: 'process-img' ' check-img' 'request' 'response'; } .process-img { @@ -52,3 +53,7 @@ min-width: 100%; } } + +.mat-expansion-panel-header-title { + font-size: $text-md; +} diff --git a/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.ts b/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.ts index 6acd59cf18..5d81e5c5d2 100644 --- a/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.ts +++ b/ui/src/app/features/face-services/face-verification/verification-result/verification-result.component.ts @@ -47,6 +47,7 @@ export class VerificationResultComponent implements OnChanges { @Input() isLoaded: boolean; @Input() pending: boolean; @Input() type: ServiceTypes; + @Input() maxImageSize: number; @Output() selectProcessFile = new EventEmitter(); @Output() selectCheckFile = new EventEmitter(); diff --git a/ui/src/app/features/footer/footer.component.scss b/ui/src/app/features/footer/footer.component.scss index d89fdbcc4d..14adb10fc7 100644 --- a/ui/src/app/features/footer/footer.component.scss +++ b/ui/src/app/features/footer/footer.component.scss @@ -17,29 +17,30 @@ @import 'colors.scss'; @import 'media.scss'; -:host{ +:host { position: fixed; width: 100%; bottom: 0; } -.mat-toolbar{ +.mat-toolbar { height: 50px; - min-height: 50px + min-height: 50px; + background: $super-light-gray; } -.mat-toolbar-row{ - justify-content: flex-end +.mat-toolbar-row { + justify-content: flex-end; } -.mat-button-focus-overlay{ +.mat-button-focus-overlay { background: none; } -.mat-button{ +.mat-button { text-decoration: underline; - &:hover{ + &:hover { color: $hovered; } @@ -48,6 +49,3 @@ min-width: 55px; } } - - - diff --git a/ui/src/app/features/image-holder-collection/image-holder-collection.component.html b/ui/src/app/features/image-holder-collection/image-holder-collection.component.html new file mode 100644 index 0000000000..928a50a031 --- /dev/null +++ b/ui/src/app/features/image-holder-collection/image-holder-collection.component.html @@ -0,0 +1,27 @@ + +
+
+ + +
+
diff --git a/ui/src/app/features/image-holder-collection/image-holder-collection.component.scss b/ui/src/app/features/image-holder-collection/image-holder-collection.component.scss new file mode 100644 index 0000000000..f5f5bf0f00 --- /dev/null +++ b/ui/src/app/features/image-holder-collection/image-holder-collection.component.scss @@ -0,0 +1,26 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +@import 'media.scss'; + +.image-holder-collection { + display: flex; + flex-wrap: wrap; + gap: 1rem; + + @include mobile { + justify-content: center; + } +} diff --git a/ui/src/app/features/image-holder-collection/image-holder-collection.component.ts b/ui/src/app/features/image-holder-collection/image-holder-collection.component.ts new file mode 100644 index 0000000000..ecfc5cd15b --- /dev/null +++ b/ui/src/app/features/image-holder-collection/image-holder-collection.component.ts @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output } from '@angular/core'; +import { CollectionItem } from 'src/app/data/interfaces/collection'; + +@Component({ + selector: 'image-holder-collection', + templateUrl: './image-holder-collection.component.html', + styleUrls: ['./image-holder-collection.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ImageHolderCollectionComponent { + @Input() items: CollectionItem[] = []; + @Input() selectionMode: boolean = false; + + @Output() deleteItem = new EventEmitter(); + @Output() cancelUploadItem = new EventEmitter(); + @Output() selectItem = new EventEmitter(); +} diff --git a/ui/src/app/features/image-holder-collection/image-holder-collection.module.ts b/ui/src/app/features/image-holder-collection/image-holder-collection.module.ts new file mode 100644 index 0000000000..c5688af414 --- /dev/null +++ b/ui/src/app/features/image-holder-collection/image-holder-collection.module.ts @@ -0,0 +1,26 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { ImageHolderModule } from '../image-holder/image-holder.module'; +import { ImageHolderCollectionComponent } from './image-holder-collection.component'; + +@NgModule({ + declarations: [ImageHolderCollectionComponent], + exports: [ImageHolderCollectionComponent], + imports: [CommonModule, ImageHolderModule], +}) +export class ImageHolderCollectionModule {} diff --git a/ui/src/app/features/image-holder/image-holder.component.html b/ui/src/app/features/image-holder/image-holder.component.html new file mode 100644 index 0000000000..b87310e3ef --- /dev/null +++ b/ui/src/app/features/image-holder/image-holder.component.html @@ -0,0 +1,27 @@ + +
+ +
+ +
+ +
+ +
+ + +
diff --git a/ui/src/app/features/image-holder/image-holder.component.scss b/ui/src/app/features/image-holder/image-holder.component.scss new file mode 100644 index 0000000000..508da54fdb --- /dev/null +++ b/ui/src/app/features/image-holder/image-holder.component.scss @@ -0,0 +1,85 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +@import 'mixins.scss'; +@import 'colors.scss'; + +.image-holder { + position: relative; + width: 13rem; + + .delete-icon { + position: absolute; + top: 0.5rem; + right: 0.5rem; + z-index: 1; + + button { + cursor: pointer; + border: none; + background: none; + + .mat-button-wrapper { + padding: 0.5rem 0; + } + + ::ng-deep .mat-button-wrapper { + padding: 0; + + mat-icon { + font-size: 1.2rem; + } + } + } + } + + img { + display: block; + width: 100%; + height: 150px; + object-fit: cover; + border-radius: 10px; + } + + &_selection-mode { + cursor: pointer; + } +} + +.selection-overlay { + @include wrapper-center(); + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; + backdrop-filter: blur(20px) brightness(0.6); + + input { + position: absolute; + top: 1rem; + left: 1rem; + } +} + +.selection-icon { + @include xl-icon-size(6rem); + color: $white; + opacity: 0.7; +} + +.mat-icon { + @include sm-icon-size(16px); +} diff --git a/ui/src/app/features/image-holder/image-holder.component.ts b/ui/src/app/features/image-holder/image-holder.component.ts new file mode 100644 index 0000000000..5fd419e124 --- /dev/null +++ b/ui/src/app/features/image-holder/image-holder.component.ts @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { EventEmitter, SimpleChanges } from '@angular/core'; +import { Component, Input, Output, ChangeDetectionStrategy, OnChanges } from '@angular/core'; +import { CircleLoadingProgressEnum } from 'src/app/data/enums/circle-loading-progress.enum'; +import { CollectionItem } from 'src/app/data/interfaces/collection'; + +@Component({ + selector: 'image-holder', + templateUrl: './image-holder.component.html', + styleUrls: ['./image-holder.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ImageHolderComponent implements OnChanges { + @Input() item: CollectionItem; + @Input() selectionMode: CollectionItem; + + isDeleteVisible: boolean; + + @Output() onDelete = new EventEmitter(); + @Output() onCancel = new EventEmitter(); + @Output() onSelect = new EventEmitter(); + + ngOnChanges(changes: SimpleChanges): void { + if (changes.item?.currentValue) { + this.isDeleteVisible = + this.item.status === CircleLoadingProgressEnum.Uploaded || this.item.status === CircleLoadingProgressEnum.Failed; + } + } + + onItemClick() { + if (this.selectionMode) { + this.onSelect.emit(this.item); + } + } +} diff --git a/ui/src/app/features/image-holder/image-holder.module.ts b/ui/src/app/features/image-holder/image-holder.module.ts new file mode 100644 index 0000000000..cd3f179472 --- /dev/null +++ b/ui/src/app/features/image-holder/image-holder.module.ts @@ -0,0 +1,29 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { ImageHolderComponent } from './image-holder.component'; +import { CircleLoadingProgressModule } from '../circle-loading-progress/circle-loading-progress.module'; +import { MatIconModule } from '@angular/material/icon'; +import { MatButtonModule } from '@angular/material/button'; +import { FormsModule } from '@angular/forms'; + +@NgModule({ + declarations: [ImageHolderComponent], + exports: [ImageHolderComponent], + imports: [CommonModule, MatIconModule, FormsModule, MatButtonModule, CircleLoadingProgressModule], +}) +export class ImageHolderModule {} diff --git a/ui/src/app/features/invite-dialog/invite-dialog.component.html b/ui/src/app/features/invite-dialog/invite-dialog.component.html index ce2c2d508d..c9d7442f2c 100644 --- a/ui/src/app/features/invite-dialog/invite-dialog.component.html +++ b/ui/src/app/features/invite-dialog/invite-dialog.component.html @@ -14,33 +14,36 @@ ~ permissions and limitations under the License. --> -

{{ 'common.capital.add' | translate }} {{ 'users.user' | translate }}

-
-
- - {{ 'common.email' | translate }} - - {{ 'app_users.add_user_dialog.email_hint' | translate }} - +
+
+

{{ 'common.capital.add' | translate }} {{ 'users.user' | translate }}

+
+ +
+ - - - - {{ 'common.type' | translate }} +
+ + {{ 'application_user_list.required_msg' | translate }} + +
+
+
+ {{ role }} - +
{{ option }} +
+ + +
+
-
- - -
- diff --git a/ui/src/app/features/invite-dialog/invite-dialog.component.scss b/ui/src/app/features/invite-dialog/invite-dialog.component.scss new file mode 100644 index 0000000000..40b108b599 --- /dev/null +++ b/ui/src/app/features/invite-dialog/invite-dialog.component.scss @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language go verning + * permissions and limitations under the License. + */ +@import 'forms.scss'; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'sizes.scss'; + +.dialog { + &__header { + width: 100%; + &--title { + margin: 1rem 0 0; + font-size: $text-lg; + font-weight: 600; + color: $dark-blue; + } + } +} + +.form-field { + @include form-field($light-gray, $lighter-gray); +} +.actions { + margin-top: 2rem; + display: flex; + justify-content: space-between; + + .add { + display: inline-block; + @include submit-btn; + @include action-btns(184px, $white, $light-blue); + border: none; + } + + .cancel { + @include submit-btn; + @include action-btns(126px, $light-blue, $white); + background: $white; + color: $light-blue; + border: 2px solid $light-blue; + } +} diff --git a/ui/src/app/features/invite-dialog/invite-dialog.component.ts b/ui/src/app/features/invite-dialog/invite-dialog.component.ts index 40be3e2c3e..ade16ffcc1 100644 --- a/ui/src/app/features/invite-dialog/invite-dialog.component.ts +++ b/ui/src/app/features/invite-dialog/invite-dialog.component.ts @@ -23,6 +23,7 @@ import { EMAIL_REGEXP_PATTERN } from 'src/app/core/constants'; @Component({ selector: 'app-invite-dialog', templateUrl: './invite-dialog.component.html', + styleUrls: ['./invite-dialog.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) export class InviteDialogComponent implements OnInit { @@ -54,10 +55,6 @@ export class InviteDialogComponent implements OnInit { } } - onCancelClick(): void { - this.dialogRef.close({}); - } - onInviteClick(): void { if (this.form.valid) { this.dialogRef.close({ diff --git a/ui/src/app/features/login-form/login-form.component.html b/ui/src/app/features/login-form/login-form.component.html index e2553b462e..4d9d200717 100644 --- a/ui/src/app/features/login-form/login-form.component.html +++ b/ui/src/app/features/login-form/login-form.component.html @@ -13,40 +13,48 @@ ~ or implied. See the License for the specific language governing ~ permissions and limitations under the License. --> -
- diff --git a/ui/src/app/features/model-clone-dialog/model-clone-dialog.component.scss b/ui/src/app/features/model-clone-dialog/model-clone-dialog.component.scss index 4ecec5854b..ab2b5df299 100644 --- a/ui/src/app/features/model-clone-dialog/model-clone-dialog.component.scss +++ b/ui/src/app/features/model-clone-dialog/model-clone-dialog.component.scss @@ -1,4 +1,4 @@ -/*! +/* * Copyright (c) 2020 the original author or authors * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -10,8 +10,34 @@ * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing + * or implied. See the License for the specific language go verning * permissions and limitations under the License. */ +@import 'forms.scss'; +@import 'mixins.scss'; +@import 'colors.scss'; +.form-field { + @include form-field($light-gray, $lighter-gray); + margin-bottom: 40px; +} + +.dialog { + &__title { + @include dialog-title; + } + + &__actions { + display: flex; + justify-content: space-between; + + .cancel-btn { + @include action-btns(149px, $light-blue, $white); + border: 2px solid $light-blue; + } + .clone-btn { + @include action-btns(149px, $white, $light-blue); + } + } +} diff --git a/ui/src/app/features/model-clone-dialog/model-clone-dialog.component.ts b/ui/src/app/features/model-clone-dialog/model-clone-dialog.component.ts index 65cec633e8..fa0a0e3cf8 100644 --- a/ui/src/app/features/model-clone-dialog/model-clone-dialog.component.ts +++ b/ui/src/app/features/model-clone-dialog/model-clone-dialog.component.ts @@ -24,10 +24,15 @@ import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; }) export class ModelCloneDialogComponent { initialName: string; + alreadyExists: boolean; constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any) {} + onChange(name): void { + this.alreadyExists = !!this.data.models.find(model => model.name === name); + } + get isCloneDisabled(): boolean { - return this.initialName ? this.initialName.length < 3 : true; + return this.initialName ? this.initialName.length < 1 : true; } } diff --git a/ui/src/app/features/model-info/model-info.component.html b/ui/src/app/features/model-info/model-info.component.html new file mode 100644 index 0000000000..64c8dbb06d --- /dev/null +++ b/ui/src/app/features/model-info/model-info.component.html @@ -0,0 +1,91 @@ + + +
+
+
+

{{ 'model_info.service_info' | translate }}

+ +
+ {{ 'model_info.service_name' | translate }} + + {{ currentModel.name | truncate: 40 }} +
+ +
+ {{ 'model_info.service_type' | translate }} + {{ currentModel.type | lowercase }} +
+ +
+ {{ 'model_info.created_on' | translate }} + {{ currentModel.createdDate | date: 'yyyy-MM-dd' }} +
+ +
+ {{ 'models.api_key' | translate }} + {{ currentModel.apiKey }} + + +
+
+ +
+

{{ 'side_menu.face_collection' | translate }}

+ +
+
+ +
+

{{ 'model_info.total_images' | translate }}

+

{{ currentModel.imageCount }}

+
+
+ +
+ +
+

{{ 'model_info.total_faces' | translate }}

+

{{ currentModel.subjectCount }}

+
+
+
+
+
+
+
+ +
+
+ +
+
+
diff --git a/ui/src/app/features/model-info/model-info.component.scss b/ui/src/app/features/model-info/model-info.component.scss new file mode 100644 index 0000000000..5c94f0a29c --- /dev/null +++ b/ui/src/app/features/model-info/model-info.component.scss @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'mixins.scss'; +@import 'colors.scss'; +@import 'sizes.scss'; +@import 'media.scss'; + +* { + color: $dark-blue; +} + +.service { + &__container { + margin-top: 1rem; + margin-bottom: 1rem; + display: flex; + } + + &__info { + padding: 30px 60px; + width: 50%; + background: $lighter-blue; + border-radius: 18px; + font-size: $text-md; + + &--title { + font-weight: bold; + } + + .label { + font-size: $text-md; + line-height: 30px; + margin-right: 1rem; + font-weight: bold; + } + + .type { + text-transform: capitalize; + } + + .api-key, + .date { + margin: 0; + font-size: $text-sm; + } + + .mat-icon { + @include sm-icon-size(); + } + } + + &__collection { + padding: 30px 60px; + font-weight: 600; + width: 50%; + background: $lighter-blue; + border-radius: 18px; + margin-left: 1rem; + + &--container { + display: flex; + font-weight: 600; + + .aside { + display: flex; + align-items: center; + margin-bottom: 1rem; + width: 50%; + + &:first-child { + padding-right: 1rem; + margin-right: 1rem; + border-right: 1px solid $lighter-gray; + } + + .content { + margin: auto; + + &--value { + font-weight: 400; + } + } + + .icon { + @include lg-icon-size(70px); + } + } + } + } + + .charts-container { + display: flex; + justify-content: space-between; + .statistics { + height: 50%; + width: 50%; + } + } +} + +@include desktop-lg { + .service { + height: 90%; + overflow-y: auto; + &__container { + display: block; + padding: 0; + } + + &__info, + &__collection { + margin: 0; + margin-bottom: 30px; + width: 100% !important; + } + } +} + +.title { + font-size: $text-md; + font-weight: bold; +} + +img { + filter: invert(12%) sepia(28%) saturate(3554%) hue-rotate(192deg) brightness(20%) contrast(102%); +} diff --git a/ui/src/app/features/model-info/model-info.component.ts b/ui/src/app/features/model-info/model-info.component.ts new file mode 100644 index 0000000000..a6a6b2b6f2 --- /dev/null +++ b/ui/src/app/features/model-info/model-info.component.ts @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { Observable, Subject, combineLatest } from 'rxjs'; +import { shareReplay, takeUntil } from 'rxjs/operators'; +import { ServiceTypes } from 'src/app/data/enums/service-types.enum'; +import { Model } from 'src/app/data/interfaces/model'; +import { Statistics } from 'src/app/data/interfaces/statistics'; +import { ModelInfoFacade } from './model-info.facade'; + +@Component({ + selector: 'model-info', + templateUrl: './model-info.component.html', + styleUrls: ['./model-info.component.scss'], +}) +export class ModelInfoComponent implements OnInit, OnDestroy { + currentModel: Model; + recognition = ServiceTypes.Recognition; + statistics$: Observable; + unsubscribe$ = new Subject(); + + constructor(private modelInfoFacade: ModelInfoFacade) { + this.statistics$ = this.modelInfoFacade.statistics$.pipe(shareReplay()); + } + + ngOnInit(): void { + combineLatest([this.modelInfoFacade.selectCurrentAppId$, this.modelInfoFacade.selectCurrentModelId$]) + .pipe(takeUntil(this.unsubscribe$)) + .subscribe(([appId, modelId]) => { + this.modelInfoFacade.loadModelStatistics(appId, modelId); + }); + + this.modelInfoFacade.statistics$.subscribe(); + + this.modelInfoFacade.currentModel$.pipe(takeUntil(this.unsubscribe$)).subscribe(model => (this.currentModel = model)); + } + + ngOnDestroy(): void { + this.unsubscribe$.next(); + this.unsubscribe$.complete(); + } +} diff --git a/ui/src/app/features/model-info/model-info.facade.ts b/ui/src/app/features/model-info/model-info.facade.ts new file mode 100644 index 0000000000..9e3eebe991 --- /dev/null +++ b/ui/src/app/features/model-info/model-info.facade.ts @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { Injectable } from '@angular/core'; +import { Store } from '@ngrx/store'; +import { Observable } from 'rxjs'; +import { filter } from 'rxjs/operators'; +import { Application } from 'src/app/data/interfaces/application'; +import { Model } from 'src/app/data/interfaces/model'; +import { Statistics } from 'src/app/data/interfaces/statistics'; +import { selectCurrentAppId } from 'src/app/store/application/selectors'; +import { loadModel, setSelectedModelIdEntityAction } from 'src/app/store/model/action'; +import { selectCurrentModel, selectCurrentModelId } from 'src/app/store/model/selectors'; +import { loadModelStatistics } from 'src/app/store/statistics/actions'; +import { selectModelStatistics } from 'src/app/store/statistics/selectors'; + +@Injectable() +export class ModelInfoFacade { + currentApp$: Observable; + currentModel$: Observable; + statistics$: Observable; + selectCurrentAppId$: Observable; + selectCurrentModelId$: Observable; + + constructor(private store: Store) { + this.currentModel$ = this.store.select(selectCurrentModel).pipe(filter(model => !!model)); + this.statistics$ = this.store.select(selectModelStatistics).pipe(filter(data => !!data[0])); + this.selectCurrentAppId$ = this.store.select(selectCurrentAppId); + this.selectCurrentModelId$ = this.store.select(selectCurrentModelId); + } + + loadModelStatistics(appId: string, modelId: string): void { + this.store.dispatch(loadModelStatistics({ appId, modelId })); + } +} diff --git a/ui/src/app/features/model-info/model-info.module.ts b/ui/src/app/features/model-info/model-info.module.ts new file mode 100644 index 0000000000..2b80227400 --- /dev/null +++ b/ui/src/app/features/model-info/model-info.module.ts @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { CommonModule } from '@angular/common'; +import { NgModule } from '@angular/core'; +import { MatButtonModule } from '@angular/material/button'; +import { MatIconModule } from '@angular/material/icon'; +import { TranslateModule } from '@ngx-translate/core'; +import { ModelInfoComponent } from './model-info.component'; +import { ModelInfoFacade } from './model-info.facade'; +import { ClipboardModule } from '@angular/cdk/clipboard'; +import { TruncateModule } from 'src/app/ui/truncate-pipe/truncate.module'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { ModelStatisticsModule } from '../model-statistics/model-statistics.module'; + +@NgModule({ + declarations: [ModelInfoComponent], + imports: [ + CommonModule, + TranslateModule, + MatButtonModule, + MatIconModule, + ClipboardModule, + MatTooltipModule, + TruncateModule, + ModelStatisticsModule, + ], + exports: [ModelInfoComponent], + providers: [ModelInfoFacade], +}) +export class ModelInfoModule {} diff --git a/ui/src/app/features/model-list/model-list-facade.ts b/ui/src/app/features/model-list/model-list-facade.ts index c0acd5301e..6e866cb8bc 100644 --- a/ui/src/app/features/model-list/model-list-facade.ts +++ b/ui/src/app/features/model-list/model-list-facade.ts @@ -16,38 +16,53 @@ import { Injectable } from '@angular/core'; import { Store } from '@ngrx/store'; import { Observable, Subscription } from 'rxjs'; +import { Application } from 'src/app/data/interfaces/application'; import { IFacade } from 'src/app/data/interfaces/IFacade'; import { Model } from 'src/app/data/interfaces/model'; import { AppState } from 'src/app/store'; -import { selectCurrentAppId } from 'src/app/store/application/selectors'; +import { loadAppUserEntityAction } from 'src/app/store/app-user/action'; +import { selectCurrentApp } from 'src/app/store/application/selectors'; import { createModel, cloneModel, deleteModel, loadModels, updateModel } from 'src/app/store/model/action'; import { selectModels, selectPendingModel, selectUserRole } from 'src/app/store/model/selectors'; +import { loadRolesEntity } from 'src/app/store/role/action'; +import { loadUsersEntity } from 'src/app/store/user/action'; @Injectable() export class ModelListFacade implements IFacade { models$: Observable; isLoading$: Observable; userRole$: Observable; - selectedApplication$: Observable; + selectedApplication$: Observable; selectedApplicationId: string; private selectedApplicationSubscription: Subscription; constructor(private store: Store) { this.models$ = this.store.select(selectModels); this.isLoading$ = this.store.select(selectPendingModel); - this.selectedApplication$ = this.store.select(selectCurrentAppId); + this.selectedApplication$ = this.store.select(selectCurrentApp); this.userRole$ = this.store.select(selectUserRole); } initSubscriptions(): void { this.selectedApplicationSubscription = this.selectedApplication$.subscribe(result => { - if (result !== null) { - this.selectedApplicationId = result; + if (result) { + this.selectedApplicationId = result.id; this.loadModels(); + this.loadData(); } }); } + loadData(): void { + this.store.dispatch( + loadAppUserEntityAction({ + applicationId: this.selectedApplicationId, + }) + ); + this.store.dispatch(loadRolesEntity()); + this.store.dispatch(loadUsersEntity()); + } + loadModels(): void { this.store.dispatch( loadModels({ diff --git a/ui/src/app/features/model-list/model-list.component.html b/ui/src/app/features/model-list/model-list.component.html index 0ecd751c87..80cd0612be 100644 --- a/ui/src/app/features/model-list/model-list.component.html +++ b/ui/src/app/features/model-list/model-list.component.html @@ -18,6 +18,7 @@
-

{{ 'models.create_model_info' | translate }}

+

{{ 'models.create_model_info' | translate }}
diff --git a/ui/src/app/features/model-list/model-list.component.scss b/ui/src/app/features/model-list/model-list.component.scss index f89bb7aa18..ccdc99b55a 100644 --- a/ui/src/app/features/model-list/model-list.component.scss +++ b/ui/src/app/features/model-list/model-list.component.scss @@ -14,7 +14,10 @@ * permissions and limitations under the License. */ +@import 'sizes.scss'; + .model-list { + height: 100%; app-spinner { position: absolute; diff --git a/ui/src/app/features/model-list/model-list.component.ts b/ui/src/app/features/model-list/model-list.component.ts index b76977f175..1b50d8523f 100644 --- a/ui/src/app/features/model-list/model-list.component.ts +++ b/ui/src/app/features/model-list/model-list.component.ts @@ -18,17 +18,17 @@ import { MatDialog } from '@angular/material/dialog'; import { Router } from '@angular/router'; import { TranslateService } from '@ngx-translate/core'; import { Observable } from 'rxjs'; -import { filter, first, map } from 'rxjs/operators'; +import { filter, first, map, tap } from 'rxjs/operators'; import { Role } from 'src/app/data/enums/role.enum'; import { Model } from 'src/app/data/interfaces/model'; import { ITableConfig } from 'src/app/features/table/table.component'; import { Routes } from '../../data/enums/routers-url.enum'; -import { DeleteDialogComponent } from '../delete-dialog/delete-dialog.component'; import { EditDialogComponent } from '../edit-dialog/edit-dialog.component'; import { ModelListFacade } from './model-list-facade'; import { ModelCreateDialogComponent } from '../mode-create-dialog/model-create-dialog.component'; import { ModelCloneDialogComponent } from '../model-clone-dialog/model-clone-dialog.component'; +import { Application } from 'src/app/data/interfaces/application'; @Component({ selector: 'app-model-list', @@ -40,10 +40,13 @@ export class ModelListComponent implements OnInit, OnDestroy { isLoading$: Observable; userRole$: Observable; tableConfig$: Observable; + currentApp$: Observable; + models: Model[]; roleEnum = Role; columns = [ - { title: 'content', property: 'content' }, - { title: 'test', property: 'test' }, + { title: 'name', property: 'name' }, + { title: 'apiKey', property: 'apiKey' }, + { title: 'type', property: 'type' }, { title: 'actions', property: 'id' }, ]; search = ''; @@ -58,9 +61,11 @@ export class ModelListComponent implements OnInit, OnDestroy { } ngOnInit() { + this.currentApp$ = this.modelListFacade.selectedApplication$; this.isLoading$ = this.modelListFacade.isLoading$; this.userRole$ = this.modelListFacade.userRole$; this.tableConfig$ = this.modelListFacade.models$.pipe( + tap(models => (this.models = models)), map(models => ({ columns: this.columns, data: models, @@ -68,21 +73,16 @@ export class ModelListComponent implements OnInit, OnDestroy { ); } - copyApiKey(apiKey: string) { - const input = document.createElement('input'); - input.setAttribute('value', apiKey); - document.body.appendChild(input); - input.select(); - document.execCommand('copy'); - document.body.removeChild(input); - } - edit(model: Model) { const dialog = this.dialog.open(EditDialogComponent, { panelClass: 'custom-mat-dialog', data: { - entityType: this.translate.instant('models.header'), + type: this.translate.instant('models.header'), + label: this.translate.instant('models.type'), + errorMsg: this.translate.instant('models.error_msg'), + models: this.models, entityName: model.name, + entityType: model.type, }, }); @@ -90,32 +90,21 @@ export class ModelListComponent implements OnInit, OnDestroy { .afterClosed() .pipe( first(), - filter(name => name) + filter(res => res), + tap(res => { + res.update ? this.modelListFacade.renameModel(model.id, res.name) : this.modelListFacade.deleteModel(model.id); + }) ) - .subscribe(name => this.modelListFacade.renameModel(model.id, name)); + .subscribe(); } clone(model: Model) { const dialog = this.dialog.open(ModelCloneDialogComponent, { - panelClass: 'custom-mat-dialog', - data: { entityType: this.translate.instant('models.header') }, - }); - - dialog - .afterClosed() - .pipe( - first(), - filter(name => name) - ) - .subscribe(name => this.modelListFacade.cloneModel(model.id, name)); - } - - delete(model: Model) { - const dialog = this.dialog.open(DeleteDialogComponent, { panelClass: 'custom-mat-dialog', data: { entityType: this.translate.instant('models.header'), - entityName: model.name, + errorMsg: this.translate.instant('models.error_msg'), + models: this.models, }, }); @@ -123,9 +112,9 @@ export class ModelListComponent implements OnInit, OnDestroy { .afterClosed() .pipe( first(), - filter(result => result) + filter(name => name) ) - .subscribe(() => this.modelListFacade.deleteModel(model.id)); + .subscribe(name => this.modelListFacade.cloneModel(model.id, name)); } test(model: Model) { @@ -138,11 +127,22 @@ export class ModelListComponent implements OnInit, OnDestroy { }); } - mangeCollection(model: Model) { + mangeCollection(model: Model): void { this.router.navigate([Routes.ManageCollection], { queryParams: { app: this.modelListFacade.selectedApplicationId, model: model.id, + type: model.type, + }, + }); + } + + dashboard(model: Model): void { + this.router.navigate([Routes.Dashboard], { + queryParams: { + app: this.modelListFacade.selectedApplicationId, + model: model.id, + type: model.type, }, }); } @@ -152,6 +152,8 @@ export class ModelListComponent implements OnInit, OnDestroy { panelClass: 'custom-mat-dialog', data: { entityType: this.translate.instant('models.header'), + errorMsg: this.translate.instant('models.error_msg'), + models: this.models, }, }); diff --git a/ui/src/app/features/model-statistics/model-statistics.component.html b/ui/src/app/features/model-statistics/model-statistics.component.html new file mode 100644 index 0000000000..ce72d4dca4 --- /dev/null +++ b/ui/src/app/features/model-statistics/model-statistics.component.html @@ -0,0 +1,36 @@ + +
+ + + + + + +

+ {{ 'models.statistics_data_msg' | translate }} +

+ +
diff --git a/ui/src/app/features/model-statistics/model-statistics.component.scss b/ui/src/app/features/model-statistics/model-statistics.component.scss new file mode 100644 index 0000000000..8b48b1d29b --- /dev/null +++ b/ui/src/app/features/model-statistics/model-statistics.component.scss @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +@import 'colors.scss'; + +p { + color: $pale-blue; + text-align: center; +} diff --git a/ui/src/app/features/model-statistics/model-statistics.component.ts b/ui/src/app/features/model-statistics/model-statistics.component.ts new file mode 100644 index 0000000000..d530b5e466 --- /dev/null +++ b/ui/src/app/features/model-statistics/model-statistics.component.ts @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { Component, Input, OnChanges } from '@angular/core'; +import { ChartDataSets, ChartOptions } from 'chart.js'; +import { Label, Color } from 'ng2-charts'; +import { Statistics } from 'src/app/data/interfaces/statistics'; +import { ModelStatsService } from './model-statistics.service'; + +@Component({ + selector: 'model-statistics', + templateUrl: './model-statistics.component.html', + styleUrls: ['./model-statistics.component.scss'], +}) +export class ModelStatisticsComponent implements OnChanges { + @Input() statistics: Statistics[]; + @Input() type: string; + + labels: Label[]; + + // Array of different segments in chart + lineChartData: ChartDataSets[] = [{ data: [] }]; + + // Define chart options + lineChartOptions: ChartOptions = { + responsive: true, + legend: { + labels: { + fontFamily: "'Poppins', sans-serif", + }, + }, + }; + + // Define colors of chart segments + lineChartColors: Color[] = [ + { + backgroundColor: 'rgba(140,151,167,0.5)', + borderColor: 'rgb(4,35,77)', + }, + ]; + + constructor(private statsService: ModelStatsService) {} + + ngOnChanges(): void { + this.labels = this.type === 'month' ? this.getMonth(Object.values(this.statistics)) : this.getDays(Object.values(this.statistics)); + } + + getMonth(stats): Label[] { + const monthStats = this.statsService.getMonthStats(stats); + this.lineChartData = [{ data: monthStats.map(el => el.requestCount) }]; + return monthStats.map(el => el.createdDate); + } + + getDays(stats: Statistics[]): Label[] { + const dayStats = this.statsService.getDays(stats); + this.lineChartData = [{ data: dayStats.map(el => el.requestCount) }]; + return dayStats.map(el => el.createdDate); + } +} diff --git a/ui/src/app/features/model-statistics/model-statistics.module.ts b/ui/src/app/features/model-statistics/model-statistics.module.ts new file mode 100644 index 0000000000..16b1e810f1 --- /dev/null +++ b/ui/src/app/features/model-statistics/model-statistics.module.ts @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { CommonModule } from '@angular/common'; +import { NgModule } from '@angular/core'; +import { TranslateModule } from '@ngx-translate/core'; +import { ChartsModule } from 'ng2-charts'; +import { ModelStatisticsComponent } from './model-statistics.component'; +import { ModelStatsService } from './model-statistics.service'; + +@NgModule({ + declarations: [ModelStatisticsComponent], + imports: [CommonModule, TranslateModule, ChartsModule], + exports: [ModelStatisticsComponent], + providers: [ModelStatsService], +}) +export class ModelStatisticsModule {} diff --git a/ui/src/app/features/model-statistics/model-statistics.service.ts b/ui/src/app/features/model-statistics/model-statistics.service.ts new file mode 100644 index 0000000000..a6a538e43a --- /dev/null +++ b/ui/src/app/features/model-statistics/model-statistics.service.ts @@ -0,0 +1,148 @@ +import { Injectable } from '@angular/core'; +import { TranslateService } from '@ngx-translate/core'; +import { Statistics } from 'src/app/data/interfaces/statistics'; + +interface Month { + name: string; + maxDays: number; +} + +@Injectable() +export class ModelStatsService { + months: Month[] = [ + { + name: this.translate.instant('month.January'), + maxDays: 31, + }, + { + name: this.translate.instant('month.February'), + maxDays: 29, + }, + { + name: this.translate.instant('month.March'), + maxDays: 31, + }, + { + name: this.translate.instant('month.April'), + maxDays: 30, + }, + { + name: this.translate.instant('month.May'), + maxDays: 31, + }, + { + name: this.translate.instant('month.June'), + maxDays: 30, + }, + { + name: this.translate.instant('month.July'), + maxDays: 31, + }, + { + name: this.translate.instant('month.August'), + maxDays: 31, + }, + { + name: this.translate.instant('month.September'), + maxDays: 30, + }, + { + name: this.translate.instant('month.October'), + maxDays: 31, + }, + { + name: this.translate.instant('month.November'), + maxDays: 30, + }, + { + name: this.translate.instant('month.December'), + maxDays: 31, + }, + ]; + + constructor(private translate: TranslateService) {} + + getMonthStats(dataArr: Statistics[]) { + const newDataArr = dataArr.slice(0, dataArr.length - 1).map(el => { + const month = new Date(el.createdDate).getMonth(); + return { + requestCount: el.requestCount, + createdDate: this.months[month].name, + }; + }); + + const currentMonth = new Date().getMonth(); + const maxMonthDisplayed = 6; + + let newData = []; + + for (let i = currentMonth; i > currentMonth - maxMonthDisplayed; i--) { + const index = i < 0 ? this.months.length + i : i; // check if current month index is a negative number + const data = newDataArr.filter(el => el.createdDate === this.months[index].name); + const statData = data.length + ? data.reduce((el1, el2) => { + return { createdDate: el1.createdDate, requestCount: el1.requestCount + el2.requestCount }; + }) + : { createdDate: this.months[index].name, requestCount: 0 }; + + newData.unshift(statData); + } + + return newData; + } + + getDays(statsData: Statistics[]): Statistics[] { + const now = new Date(); + const currentData = new Date().getDate(); + const month = new Date().getMonth(); + let lastThirtyDaysArr = []; + + // gets day-month array + for (let i = 0; i < 30; i++) { + const currentMonthDate = new Date().getDate() - i; + if (currentMonthDate < 1) { + const date = this.months[month - 1].maxDays + currentMonthDate; + const monthData = month.toString().length > 1 ? month.toString() : '0' + month; + + lastThirtyDaysArr.push(date.toString().length > 1 ? `${date}-${monthData}` : `0${date}-${monthData}`); + } else { + const currentMonth = month + 1; + const monthData = currentMonth.toString().length > 1 ? currentMonth.toString() : '0' + currentMonth; + + lastThirtyDaysArr.push( + currentMonthDate.toString().length > 1 ? `${currentMonthDate}-${monthData}` : `0${currentMonthDate}-${monthData}` + ); + } + } + + // gets requestCount and createdDate data array + const dayStats = statsData + .filter(el => el.createdDate) + .map(el => { + return { + requestCount: el.requestCount, + createdDate: el.createdDate.slice(5, 10).split('-').reverse().join('-'), + }; + }); + + // gets final result to display in charts + const daysStatsData = lastThirtyDaysArr.map(el => { + const filteredDatesArr = dayStats.filter(day => el == day.createdDate); + + if (filteredDatesArr.length > 0) { + const finalResult = filteredDatesArr.reduce((el1, el2) => { + return { + requestCount: el1.requestCount + el2.requestCount, + createdDate: el1.createdDate, + }; + }); + + return finalResult; + } else { + return { requestCount: 0, createdDate: el }; + } + }); + + return daysStatsData.reverse(); + } +} diff --git a/ui/src/app/features/model-table/model-sort.pipe.ts b/ui/src/app/features/model-table/model-sort.pipe.ts new file mode 100644 index 0000000000..229159b3e2 --- /dev/null +++ b/ui/src/app/features/model-table/model-sort.pipe.ts @@ -0,0 +1,41 @@ +/*! + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { PipeTransform } from '@angular/core'; +import { Pipe } from '@angular/core'; +import { ServiceTypes } from 'src/app/data/enums/service-types.enum'; +import { Model } from 'src/app/data/interfaces/model'; + +@Pipe({ + name: 'modelSort', +}) +export class ModelSortPipe implements PipeTransform { + transform(modelCollection: Model[]) { + const recognition = modelCollection + .filter(model => model.type === ServiceTypes.Recognition) + .sort((model, next) => model.name.localeCompare(next.name)); + + const detection = modelCollection + .filter(model => model.type === ServiceTypes.Detection) + .sort((model, next) => model.name.localeCompare(next.name)); + + const verification = modelCollection + .filter(model => model.type === ServiceTypes.Verification) + .sort((model, next) => model.name.localeCompare(next.name)); + + return [...recognition, ...detection, ...verification]; + } +} diff --git a/ui/src/app/features/model-table/model-table.component.html b/ui/src/app/features/model-table/model-table.component.html index e2a8984d73..c99d6cd835 100644 --- a/ui/src/app/features/model-table/model-table.component.html +++ b/ui/src/app/features/model-table/model-table.component.html @@ -13,71 +13,110 @@ ~ or implied. See the License for the specific language governing ~ permissions and limitations under the License. --> - - +
+

{{ 'models.name' | translate }}

+

{{ 'models.api_key' | translate }}

+

{{ 'models.type' | translate }}

+

{{ 'models.actions' | translate }}

+
+
- - + + + - + + + +
{{ 'models.caption' | translate }}
-
- - {{ element.name }} - - - {{ getKeyType(element.type) | titlecase }} + +
+
+ + {{ element.name | truncate: 20 }}
-
+
+
- API-Key: {{ element.apiKey }} +
-
- +
+
+ {{ element.type | lowercase }} +
+
- - - - - - - - - - - + + + + + + +
-
+
-

{{ 'users.search.no_results' | translate }}

+

{{ 'users.search.no_results' | translate }}

diff --git a/ui/src/app/features/model-table/model-table.component.scss b/ui/src/app/features/model-table/model-table.component.scss index bbaf9bfd99..82927458c2 100644 --- a/ui/src/app/features/model-table/model-table.component.scss +++ b/ui/src/app/features/model-table/model-table.component.scss @@ -14,99 +14,100 @@ * permissions and limitations under the License. */ -@import "colors.scss"; -@import "mixins.scss"; +@import 'colors.scss'; +@import 'mixins.scss'; +@import 'sizes.scss'; + +* { + color: $dark-blue; + font-size: $text-sm; +} caption { display: none; } +:host { + height: 100%; + max-height: 600px; +} -.mat { - - &-column { - - &-content { - width: 100%; - - .content--wrapper { - display: flex; - align-items: center; - - &_name { - @include not-break-string(); - } - - &_type { - margin-left: 8px; - } - - &_key { - margin-right: 4px; - @include not-break-string(); - - .title { - color: $light-gray; - } - - .key { - color: $black; - } - } - - &_btn { - width: 24px; - height: 24px; - line-height: 24px; +.thead-container { + width: 100%; + display: flex; + p { + font-weight: bold; + font-size: $text-md; + color: $light-blue; + } - .mat-icon { - @include sm-icon-size(); - } - } + .name { + width: 30%; + } + .api-key { + width: 35%; + } + .type { + width: 10%; + } + .actions { + text-align: end; + width: 25%; + padding-right: 20px; + } +} - .recognition { - color: $dark-blue; - } +table { + height: 100%; + overflow: scroll; - .detection { - color: $orange; - } + .mat-column-name { + cursor: pointer; + width: 30%; + font-weight: bold; + } - .verify { - color: $purple; - } + .mat-column-apiKey { + width: 35%; + button { + width: 26px; + height: 26px; + display: inline-flex; + justify-content: center; + align-items: center; + .mat-icon { + @include sm-icon-size(14px); } } + } - &-test { - text-align: center; - width: 58px; + .mat-column-type { + width: 10%; + text-transform: capitalize; + font-weight: bold; + } - .test { + .mat-column-action { + width: 25%; + } - &--btn { - min-width: 0; - font-size: 12px; - line-height: 14px; - padding: 5px 14px; - border-radius: 2px; - } - } - } + .actions { + display: flex; + justify-content: flex-end; - &-actions { - text-align: center; - width: 40px; + button { + display: flex; + align-items: center; + justify-content: center; + } + } - .actions--btn { - display: flex; - align-items: center; - justify-content: center; - width: 40px; + .mat-icon { + @include sm-icon-size(17px); + } +} - .mat-icon { - @include md-icon-size(); - } - } - } +.no-data-message { + p { + @include no-data-msg; } } diff --git a/ui/src/app/features/model-table/model-table.component.ts b/ui/src/app/features/model-table/model-table.component.ts index ab62086b1b..2f39ca5626 100644 --- a/ui/src/app/features/model-table/model-table.component.ts +++ b/ui/src/app/features/model-table/model-table.component.ts @@ -27,17 +27,13 @@ import { TableComponent } from '../table/table.component'; }) export class ModelTableComponent extends TableComponent { @Input() userRole: string; - @Output() copyApiKey = new EventEmitter(); @Output() clone = new EventEmitter(); @Output() edit = new EventEmitter(); @Output() delete = new EventEmitter(); @Output() test = new EventEmitter(); + @Output() dashboard = new EventEmitter(); @Output() collection = new EventEmitter(); roleEnum = Role; types = ServiceTypes; - - getKeyType(type: string): string { - return Object.keys(this.types).find(key => this.types[key] === type) || ''; - } } diff --git a/ui/src/app/features/model-table/model-table.module.ts b/ui/src/app/features/model-table/model-table.module.ts index e60a798373..f927259f3b 100644 --- a/ui/src/app/features/model-table/model-table.module.ts +++ b/ui/src/app/features/model-table/model-table.module.ts @@ -32,15 +32,18 @@ import { MatOptionModule } from '@angular/material/core'; import { MatSelectModule } from '@angular/material/select'; import { MatDialogModule } from '@angular/material/dialog'; import { MatInputModule } from '@angular/material/input'; +import { ModelSortPipe } from './model-sort.pipe'; +import { ClipboardModule } from '@angular/cdk/clipboard'; @NgModule({ - declarations: [ModelTableComponent, ModelCreateDialogComponent, ModelCloneDialogComponent], + declarations: [ModelTableComponent, ModelSortPipe, ModelCreateDialogComponent, ModelCloneDialogComponent], exports: [ModelTableComponent], imports: [ CommonModule, MatButtonModule, MatTableModule, MatIconModule, + MatTooltipModule, MatMenuModule, TruncateModule, MatTooltipModule, @@ -51,6 +54,7 @@ import { MatInputModule } from '@angular/material/input'; MatSelectModule, MatDialogModule, MatInputModule, + ClipboardModule, ], }) export class ModelTableModule {} diff --git a/ui/src/app/features/password-recovery-dialog/password-recovery.component.html b/ui/src/app/features/password-recovery-dialog/password-recovery.component.html new file mode 100644 index 0000000000..5e29de97b0 --- /dev/null +++ b/ui/src/app/features/password-recovery-dialog/password-recovery.component.html @@ -0,0 +1,46 @@ + + +
+
+

{{ 'recovery.password_recovery' | translate }}

+
+
+
+ + +
+
+ + +
+
+ +
diff --git a/ui/src/app/features/password-recovery-dialog/password-recovery.component.scss b/ui/src/app/features/password-recovery-dialog/password-recovery.component.scss new file mode 100644 index 0000000000..76353d55ee --- /dev/null +++ b/ui/src/app/features/password-recovery-dialog/password-recovery.component.scss @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language go verning + * permissions and limitations under the License. + */ + +@import 'forms.scss'; +@import 'mixins.scss'; +@import 'colors.scss'; + +.form-field { + @include form-field($light-gray, $lighter-gray); + margin-bottom: 40px; +} + +.dialog { + &__header { + width: 100%; + &--title { + margin: 0; + font-size: 30px; + line-height: 40px; + font-weight: 600; + color: $dark-blue; + } + } + + &__content { + box-sizing: border-box; + overflow: hidden; + font-size: 23px; + color: $secondary; + } + + &__actions { + display: flex; + justify-content: space-between; + + .cancel-btn { + @include action-btns(149px, $light-blue, $white); + border: 2px solid $light-blue; + } + .save-btn { + @include action-btns(149px, $white, $light-blue); + } + } +} diff --git a/ui/src/app/features/password-recovery-dialog/password-recovery.component.ts b/ui/src/app/features/password-recovery-dialog/password-recovery.component.ts new file mode 100644 index 0000000000..9d38b01dd4 --- /dev/null +++ b/ui/src/app/features/password-recovery-dialog/password-recovery.component.ts @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { Component } from '@angular/core'; +import { MatDialogRef } from '@angular/material/dialog'; + +@Component({ + selector: 'password-recovery', + templateUrl: './password-recovery.component.html', + styleUrls: ['./password-recovery.component.scss'], +}) +export class PasswordRecoveryDialogComponent { + email: string; + + constructor(public dialogRef: MatDialogRef) {} + + onSubmit() { + this.dialogRef.close(); + } +} diff --git a/ui/src/app/features/side-menu/side-menu.component.html b/ui/src/app/features/side-menu/side-menu.component.html new file mode 100644 index 0000000000..74609c1dab --- /dev/null +++ b/ui/src/app/features/side-menu/side-menu.component.html @@ -0,0 +1,70 @@ + + +
+ + + + + + {{ 'side_menu.dashboard' | translate }} + + + + + + + + {{ 'side_menu.test' | translate }} + + + + + + + + {{ 'side_menu.face_collection' | translate }} + + + + + + +
diff --git a/ui/src/app/features/side-menu/side-menu.component.scss b/ui/src/app/features/side-menu/side-menu.component.scss new file mode 100644 index 0000000000..0875e87d71 --- /dev/null +++ b/ui/src/app/features/side-menu/side-menu.component.scss @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +@import 'colors.scss'; +@import 'sizes.scss'; + +.menu { + position: relative; + height: 100%; + background: linear-gradient($primary 0%, $primary-lighter 100%); + color: $white; + font-size: $text-md; + font-weight: bold; + display: flex; + flex-direction: column; + justify-content: space-between; + margin-right: 38px; + + .toggle-list { + display: flex; + justify-content: center; + align-items: center; + cursor: pointer; + margin-bottom: 10px; + width: 50px; + border: none; + background: none; + + .mat-icon { + display: flex; + align-items: center; + justify-content: center; + text-align: center; + width: 100%; + font-size: 14px; + width: 31px; + height: 25px; + color: $white; + } + } + + .mat-nav-list { + cursor: pointer; + margin: 5px; + display: flex; + + .mat-icon { + text-align: center; + width: 25px; + height: 30px; + margin: 0 13px; + } + + a { + display: flex; + align-items: center; + } + } +} diff --git a/ui/src/app/features/side-menu/side-menu.component.ts b/ui/src/app/features/side-menu/side-menu.component.ts new file mode 100644 index 0000000000..bc2c00a921 --- /dev/null +++ b/ui/src/app/features/side-menu/side-menu.component.ts @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { MatDialog } from '@angular/material/dialog'; +import { ActivatedRoute, Router } from '@angular/router'; +import { TranslateService } from '@ngx-translate/core'; +import { Subscription } from 'rxjs'; +import { map, tap } from 'rxjs/operators'; +import { CircleLoadingProgressEnum } from 'src/app/data/enums/circle-loading-progress.enum'; +import { Routes } from 'src/app/data/enums/routers-url.enum'; +import { ServiceTypes } from 'src/app/data/enums/service-types.enum'; +import { ConfirmDialogComponent } from '../confirm-dialog/confirm-dialog.component'; +import { SideMenuFacade } from './side-menu.facade'; + +@Component({ + selector: 'side-menu', + templateUrl: './side-menu.component.html', + styleUrls: ['./side-menu.component.scss'], +}) +export class SideMenuComponent implements OnInit, OnDestroy { + app: string; + model: string; + type: string; + closed: boolean = false; + recognition: string = ServiceTypes.Recognition; + currentPage: string; + itemsInProgress: boolean; + subs: Subscription; + + constructor( + private router: Router, + private route: ActivatedRoute, + private dialog: MatDialog, + private sideMenuFacade: SideMenuFacade, + private translate: TranslateService + ) { + this.subs = this.sideMenuFacade.collectionItems$ + .pipe( + map(collection => !!collection.find(item => item.status === CircleLoadingProgressEnum.InProgress)), + tap(res => (this.itemsInProgress = res)) + ) + .subscribe(); + } + + ngOnInit() { + this.app = this.route.snapshot.queryParams.app; + this.model = this.route.snapshot.queryParams.model; + this.type = this.route.snapshot.queryParams.type; + this.currentPage = '/' + this.route.snapshot.url[0]?.path; + } + + get routes(): typeof Routes { + return Routes; + } + + onDashBoard(path: string): void { + const queryParams = { + app: this.app, + model: this.model, + type: this.type, + }; + + this.itemsInProgress + ? this.openDialog(path, queryParams) + : this.router.navigate([path], { + queryParams: queryParams, + }); + } + + onTest(path: string): void { + const queryParams = { + app: this.app, + model: this.model, + type: this.type, + }; + + this.itemsInProgress + ? this.openDialog(path, queryParams) + : this.router.navigate([path], { + queryParams: queryParams, + }); + } + + onManageCollection(path: string): void { + this.router.navigate([path], { + queryParams: { + app: this.app, + model: this.model, + type: this.type, + }, + }); + } + + openDialog(path: string, queryParams: { app: string; model: string; type: string }): void { + const dialog = this.dialog.open(ConfirmDialogComponent, { + panelClass: 'custom-mat-dialog', + data: { + title: this.translate.instant('org_users.confirm_dialog.title'), + description: this.translate.instant('org_users.confirm_dialog.confirmation_question'), + }, + }); + + dialog.afterClosed().subscribe(confirm => { + if (!confirm) return; + this.router.navigate([path], { queryParams: queryParams }); + }); + } + + ngOnDestroy(): void { + this.subs.unsubscribe(); + } +} diff --git a/ui/src/app/features/side-menu/side-menu.facade.ts b/ui/src/app/features/side-menu/side-menu.facade.ts new file mode 100644 index 0000000000..e765fe619a --- /dev/null +++ b/ui/src/app/features/side-menu/side-menu.facade.ts @@ -0,0 +1,15 @@ +import { Injectable } from '@angular/core'; +import { Store } from '@ngrx/store'; +import { Observable } from 'rxjs'; +import { CollectionItem } from 'src/app/data/interfaces/collection'; +import { AppState } from 'src/app/store'; +import { selectImageCollection } from 'src/app/store/manage-collectiom/selectors'; + +@Injectable() +export class SideMenuFacade { + collectionItems$: Observable; + + constructor(private store: Store) { + this.collectionItems$ = this.store.select(selectImageCollection); + } +} diff --git a/ui/src/app/features/side-menu/side-menu.module.ts b/ui/src/app/features/side-menu/side-menu.module.ts new file mode 100644 index 0000000000..7e0f48668e --- /dev/null +++ b/ui/src/app/features/side-menu/side-menu.module.ts @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { MatIconModule } from '@angular/material/icon'; +import { TranslateModule } from '@ngx-translate/core'; +import { SideMenuComponent } from './side-menu.component'; +import { MatListModule } from '@angular/material/list'; +import { MatExpansionModule } from '@angular/material/expansion'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { SideMenuFacade } from './side-menu.facade'; + +@NgModule({ + declarations: [SideMenuComponent], + exports: [SideMenuComponent], + imports: [CommonModule, MatIconModule, MatListModule, MatExpansionModule, MatTooltipModule, TranslateModule], + providers: [SideMenuFacade], +}) +export class SideMenuModule {} diff --git a/ui/src/app/features/sign-up-form/sign-up-form.component.html b/ui/src/app/features/sign-up-form/sign-up-form.component.html index a956a7d5a2..1de1774056 100644 --- a/ui/src/app/features/sign-up-form/sign-up-form.component.html +++ b/ui/src/app/features/sign-up-form/sign-up-form.component.html @@ -13,77 +13,109 @@ ~ or implied. See the License for the specific language governing ~ permissions and limitations under the License. --> -