Merge branch 'rc/1.2.0' into stable

# Conflicts: # scripts/packages/tator-js
cvisionai · Nov 4, 2023 · 3afeecb · 3afeecb
2 parents ae51d6a + 7380e8d
commit 3afeecb
Show file tree

Hide file tree

Showing 89 changed files with 3,293 additions and 794 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -131,8 +131,8 @@ jobs:
           ssh lightsail 'python3 get-pip.py';
           ssh lightsail '/home/ubuntu/.local/bin/pip3 install --upgrade awscli';
           echo 'Initiating lightsail self-destruct sequence...';
-          ssh lightsail 'export AWS_ACCESS_KEY_ID='"'$AWS_ACCESS_KEY_ID'"';export AWS_SECRET_ACCESS_KEY='"'$AWS_SECRET_ACCESS_KEY'"';export AWS_DEFAULT_REGION='"'$AWS_DEFAULT_REGION'"';export GIT_REVISION='"'$CIRCLE_SHA1'"';sh -c "sleep 10800 && /home/ubuntu/.local/bin/aws lightsail delete-instance --instance-name tator-ci-$GIT_REVISION" >/dev/null 2>&1 &';
-          ssh lightsail 'echo "This lightsail instance will self-destruct in 3 hours."';
+          ssh lightsail 'export AWS_ACCESS_KEY_ID='"'$AWS_ACCESS_KEY_ID'"';export AWS_SECRET_ACCESS_KEY='"'$AWS_SECRET_ACCESS_KEY'"';export AWS_DEFAULT_REGION='"'$AWS_DEFAULT_REGION'"';export GIT_REVISION='"'$CIRCLE_SHA1'"';sh -c "sleep 14400 && /home/ubuntu/.local/bin/aws lightsail delete-instance --instance-name tator-ci-$GIT_REVISION" >/dev/null 2>&1 &';
+          ssh lightsail 'echo "This lightsail instance will self-destruct in 4 hours."';
     - run:
         name: Clone source on lightsail
         command: |
@@ -155,7 +155,7 @@ jobs:
           ssh lightsail 'wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb';
           ssh lightsail 'sudo -E apt-get -yq --no-install-suggests --no-install-recommends install ./google-chrome-stable_current_amd64.deb';
           ssh lightsail 'sudo -E apt-get update && sudo -E apt-get -yq --no-install-suggests --no-install-recommends install tesseract-ocr python3-pip ffmpeg wget unzip';
-          ssh lightsail 'pip3 install pytest pytest-xdist pandas playwright==1.27.1 pytest-playwright==0.3.0 pytesseract==0.3.9 opencv-python pytest-rerunfailures==10.2';
+          ssh lightsail 'pip3 install pytest pytest-xdist pandas playwright==1.37.0 pytest-playwright==0.4.2 pytesseract==0.3.9 opencv-python pytest-rerunfailures==10.2';
           ssh lightsail 'export PATH=$PATH:$HOME/.local/bin:/snap/bin && playwright install';
           ssh lightsail 'wget http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip';
           ssh lightsail 'unzip Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip';
@@ -187,7 +187,13 @@ jobs:
         command: ssh lightsail 'cd tator && make testinit'
     - run:
         name: Run REST tests
-        command: ssh lightsail 'cd tator && for i in $(seq 1 3); do make rq-empty && make test && s=0 && break || s=$? && sleep 10; done; (exit $s)'
+        command: |
+          mkdir -p rest-test-results;
+          ssh lightsail 'cd tator && for i in $(seq 1 3); do make rq-empty && make test && break || sleep 10; done;';
+          ssh lightsail 'test -e tator/rest-junit.xml' && scp lightsail:tator/rest-junit.xml rest-test-results/ || exit 0;
+          [ $(sed -n '1s/.*failures="\([0-9]*\)".*/\1/p' rest-test-results/rest-junit.xml) -eq "0" ] && exit 0 || exit 1;
+    - store_test_results:
+        path: rest-test-results
   front-end-tests:
     machine:
       image: ubuntu-2004:202010-01
@@ -209,9 +215,14 @@ jobs:
         command: |
           ssh lightsail 'cd tator && make rq-empty';
           mkdir -p /tmp/videos;
+          mkdir -p frontend-test-results;
           ssh lightsail 'mkdir -p /tmp/videos';
           sshfs -o default_permissions lightsail:/tmp/videos /tmp/videos;
-          ssh lightsail 'mkdir -p /tmp/videos && export PATH=$PATH:$HOME/.local/bin:/snap/bin && export PYTHONUNBUFFERED=1 && cd tator && for i in $(seq 1 3); do pytest test --slowmo 30 --base-url=http://localhost:8080 --browser=chromium --username=admin --password=admin --videos=/tmp/videos -s && s=0 && break || s=$? && sleep 10; done; (exit $s)';
+          ssh lightsail 'mkdir -p /tmp/videos && export PATH=$PATH:$HOME/.local/bin:/snap/bin && export PYTHONUNBUFFERED=1 && cd tator && for i in $(seq 1 3); do if [ -d "test-results" ]; then rm -f test-results/*; else mkdir -p test-results; fi; pytest test --slowmo 30 --base-url=http://localhost:8080 --browser=chromium --username=admin --password=admin --videos=/tmp/videos -s --junitxml=test-results/frontend-junit.xml && break || sleep 10; done;';
+          ssh lightsail 'test -e tator/test-results/frontend-junit.xml' && scp lightsail:tator/test-results/frontend-junit.xml frontend-test-results/ || exit 0;
+          [ $(sed -n '1s/.*failures="\([0-9]*\)".*/\1/p' frontend-test-results/frontend-junit.xml) -eq "0" ] && exit 0 || exit 1;
+    - store_test_results:
+        path: frontend-test-results
     - store_artifacts:
         path: /tmp/videos
         destination: videos
@@ -232,7 +243,13 @@ jobs:
           scp lightsail:~/token.txt ~/token.txt
     - run:
         name: Run tator-py tests
-        command: ssh lightsail 'export TOKEN='"'$(cat ~/token.txt)'"';export PATH=$PATH:$HOME/.local/bin:/snap/bin && cd tator && for i in $(seq 1 3); do pytest tatorpy_test --ignore tatorpy_test/test_algorithm_launch.py --ignore tatorpy_test/test_job_cancel.py --host=http://localhost:8080 --token=$TOKEN -s --keep && s=0 && break || s=$? && sleep 10; done; (exit $s)';
+        command: |
+          mkdir -p tatorpy-test-results;
+          ssh lightsail 'export TATOR_TOKEN='"'$(cat ~/token.txt)'"'; export TATOR_HOST=http://localhost:8080; export PATH=$PATH:$HOME/.local/bin:/snap/bin && cd tator && for i in $(seq 1 3); do if [ -d "test-results" ]; then rm -f test-results/*; else mkdir -p test-results; fi; pytest tatorpy_test --ignore tatorpy_test/test_algorithm_launch.py --ignore tatorpy_test/test_job_cancel.py -s --keep --junitxml=test-results/tatorpy-junit.xml && break || sleep 10; done;';
+          ssh lightsail 'test -e tator/test-results/tatorpy-junit.xml' && scp lightsail:tator/test-results/tatorpy-junit.xml tatorpy-test-results/ || exit 0;
+          [ $(sed -n '1s/.*failures="\([0-9]*\)".*/\1/p' tatorpy-test-results/tatorpy-junit.xml) -eq "0" ] && exit 0 || exit 1;
+    - store_test_results:
+        path: tatorpy-test-results
     - run:
         name: Check db-worker logs for clean running
         command: ssh lightsail 'cd tator && make check-clean-db-logs'
@@ -245,7 +262,6 @@ jobs:
     - store_artifacts:
         path: /tmp/logs
         destination: container_logs
-        when: always
   cron-job-tests:
     machine:
       image: ubuntu-2004:202010-01

diff --git a/.gitignore b/.gitignore
@@ -40,6 +40,8 @@ helm/tator/configs/*
 **/debug.log
 **/.pylint-venv
 .env
+test-results
+rest-junit.xml
 
 # Hidden mac files
 ._.

diff --git a/Makefile b/Makefile
@@ -37,7 +37,9 @@ endif
 # Set this ENV to http://iad-ad-1.clouds.archive.ubuntu.com/ubuntu/ for
 # faster builds on Oracle OCI 
 APT_REPO_HOST ?= $(shell cat /etc/apt/sources.list | grep -E "focal main|jammy main" | grep -v cdrom | head -n1 | awk '{print $$2}')
-
+ifeq ($(APT_REPO_HOST),)
+APT_REPO_HOST=http://archive.ubuntu.com/ubuntu/
+endif
 
 #############################
 ## Help Rule + Generic targets
@@ -214,6 +216,10 @@ endif
 collect-static: webpack
 	@scripts/collect-static.sh
 
+force-static:
+	@rm -fr scripts/packages/tator-js/pkg/
+	$(MAKE) collect-static
+
 dev-push:
 	@scripts/dev-push.sh
 
@@ -251,7 +257,8 @@ testinit:
 .PHONY: test
 test:
 	docker exec gunicorn sh -c 'bash scripts/addExtensionsToInit.sh'
-	docker exec gunicorn sh -c 'pytest --ds=tator_online.settings -n 4 --reuse-db --create-db main/tests.py'
+	docker exec gunicorn sh -c 'pytest --ds=tator_online.settings -n 4 --reuse-db --create-db --junitxml=./test-results/rest-junit.xml main/tests.py'
+	docker cp gunicorn:/tator_online/test-results/rest-junit.xml rest-junit.xml
 
 .PHONY: cache_clear
 cache-clear:

diff --git a/api/main/_import_image.py b/api/main/_import_image.py
@@ -58,36 +58,23 @@ def _import_image(name, url, thumbnail_url, media_id, reference_only):
             temp_image = tempfile.NamedTemporaryFile(delete=False)
             download_file(url, temp_image.name, 5)
         image = Image.open(temp_image.name)
-        media_obj.width, media_obj.height = image.size
         image_format = image.format
 
         image = ImageOps.exif_transpose(image)
+        media_obj.width, media_obj.height = image.size
 
         # Add a png for compatibility purposes in case of HEIF or AVIF import.
-        # always make AVIF, but keep HEIF originals.
+        # always make AVIF
         if reference_only is False:
-            if image_format == "HEIF":
-                alt_image = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
-                image.save(alt_image, format="png")
-                alt_images.append(alt_image)
-                alt_formats.append("png")
-
-                alt_image = tempfile.NamedTemporaryFile(delete=False, suffix=".avif")
-                image.save(alt_image, format="avif")
-                alt_images.append(alt_image)
-                alt_formats.append("avif")
-
-            elif image_format == "AVIF":
-                alt_image = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
-                image.save(alt_image, format="png")
-                alt_images.append(alt_image)
-                alt_formats.append("png")
-            else:
-                # convert image upload to AVIF
-                alt_image = tempfile.NamedTemporaryFile(delete=False, suffix=".avif")
-                image.save(alt_image, format="avif")
-                alt_images.append(alt_image)
-                alt_formats.append("avif")
+            alt_image = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
+            image.save(alt_image, format="png", quality=100, subsampling=0)
+            alt_images.append(alt_image)
+            alt_formats.append("png")
+
+            alt_image = tempfile.NamedTemporaryFile(delete=False, suffix=".avif")
+            image.save(alt_image, format="avif", quality=100)
+            alt_images.append(alt_image)
+            alt_formats.append("avif")
 
         # Download or create the thumbnail.
         if thumbnail_url is None:
@@ -112,6 +99,7 @@ def _import_image(name, url, thumbnail_url, media_id, reference_only):
         thumb_height = thumb.height
         thumb.close()
 
+    media_obj.media_files = {}
     if reference_only and url:
         if media_obj.media_files is None:
             media_obj.media_files = {}
@@ -123,23 +111,10 @@ def _import_image(name, url, thumbnail_url, media_id, reference_only):
                 "mime": f"image/{image_format.lower()}",
             }
         ]
-    elif url:
-        if media_obj.media_files is None:
-            media_obj.media_files = {}
-        # Upload image.
-        image_key = f"{project_obj.organization.pk}/{project_obj.pk}/{media_obj.pk}/{name}"
-        tator_store.put_object(image_key, temp_image)
-        media_obj.media_files["image"] = [
-            {
-                "path": image_key,
-                "size": os.stat(temp_image.name).st_size,
-                "resolution": [media_obj.height, media_obj.width],
-                "mime": f"image/{image_format.lower()}",
-            }
-        ]
-        os.remove(temp_image.name)
-        Resource.add_resource(image_key, media_obj)
+    else:
+        media_obj.media_files["image"] = []
 
+    # Handle all formats the same way
     for alt_image, alt_format in zip(alt_images, alt_formats):
         alt_name = f"image.{alt_format}"
         if media_obj.media_files is None:

diff --git a/api/main/auth.py b/api/main/auth.py
@@ -46,10 +46,11 @@ def authenticate(self, request, username=None, password=None, **kwargs):
                 user.save()
                 lockout_lifted = user.last_failed_login + LOCKOUT_TIME
                 time_left = lockout_lifted - now
-                msg = f" *SECURITY ALERT:* Attempt to login during lockout"
-                msg += f" User={user}/{user.id}"
-                msg += f" Attempt count = {user.failed_login_count}"
-                msg += f" Lockout will be lifted in '{time_left}' at '{lockout_lifted}'"
+                msg = (
+                    f" *SECURITY ALERT:* Attempt to login during lockout: User={user}/{user.id}, "
+                    f" Attempt count = {user.failed_login_count}. Lockout will be lifted in "
+                    f"'{time_left}' at '{lockout_lifted}'"
+                )
                 Notify.notify_admin_msg(msg)
                 # Run the default password hasher once to reduce the timing
                 # difference (#20760).
@@ -59,30 +60,32 @@ def authenticate(self, request, username=None, password=None, **kwargs):
         if user.check_password(password) and self.user_can_authenticate(user):
             user.last_login = datetime.now(timezone.utc)
             if user.failed_login_count >= LOCKOUT_LIMIT:
-                msg = "Login proceeded after lock expiry"
-                msg += f" User={user}/{user.id}"
+                msg = f"Login proceeded after lock expiry User={user}/{user.id}"
                 Notify.notify_admin_msg(msg)
             user.failed_login_count = 0
             user.save()
             return user
-        else:
-            user.last_failed_login = datetime.now(timezone.utc)
-            user.failed_login_count += 1
-            user.save()
-            if user.failed_login_count >= LOCKOUT_LIMIT:
-                msg = f"*SECURITY ALERT:* Bad Login Attempt for {user}/{user.id}"
-                msg += f" Attempt count = {user.failed_login_count}"
-                Notify.notify_admin_msg(msg)
-            # Send an email if the failed login count has been reached.
-            if (user.failed_login_count == LOCKOUT_LIMIT) and settings.TATOR_EMAIL_ENABLED:
-                get_email_service().email(
-                    sender=settings.TATOR_EMAIL_SENDER,
-                    recipients=[user.email],
-                    title=f"Tator account has been locked",
-                    text="This message is to notify you that your Tator account (username "
-                    f"{user.username}) has been locked due to {LOCKOUT_LIMIT} failed logins. "
-                    "Your account will be unlocked automatically after 10 minutes, or you "
-                    "can unlock your account now by resetting your password. To reset your "
-                    "password, follow the procedure described here:\n\n"
-                    "https://tator.io/tutorials/2021-06-11-reset-your-password/",
-                )
+
+        user.last_failed_login = datetime.now(timezone.utc)
+        user.failed_login_count += 1
+        user.save()
+        if user.failed_login_count >= LOCKOUT_LIMIT:
+            msg = (
+                f"*SECURITY ALERT:* Bad Login Attempt for {user}/{user.id}. Attempt count = "
+                f"{user.failed_login_count}"
+            )
+            Notify.notify_admin_msg(msg)
+        # Send an email if the failed login count has been reached.
+        email_service = get_email_service()
+        if user.failed_login_count == LOCKOUT_LIMIT and email_service:
+            email_service.email(
+                sender=settings.TATOR_EMAIL_SENDER,
+                recipients=[user.email],
+                title=f"Tator account has been locked",
+                text="This message is to notify you that your Tator account (username "
+                f"{user.username}) has been locked due to {LOCKOUT_LIMIT} failed logins. "
+                "Your account will be unlocked automatically after 10 minutes, or you "
+                "can unlock your account now by resetting your password. To reset your "
+                "password, follow the procedure described here:\n\n"
+                "https://tator.io/tutorials/2021-06-11-reset-your-password/",
+            )
diff --git a/api/main/kube.py b/api/main/kube.py
@@ -285,10 +285,12 @@ def __init__(self, alg):
             api_client = ApiClient(conf)
             self.corev1 = CoreV1Api(api_client)
             self.custom = CustomObjectsApi(api_client)
+            self.host = f'{PROTO}{os.getenv("MAIN_HOST")}'
         else:
             load_incluster_config()
             self.corev1 = CoreV1Api()
             self.custom = CustomObjectsApi()
+            self.host = "http://gunicorn-svc:8000"
 
         # Read in the manifest.
         if alg.manifest:
@@ -348,19 +350,19 @@ def start_algorithm(
                 },
                 {
                     "name": "host",
-                    "value": f'{PROTO}{os.getenv("MAIN_HOST")}',
+                    "value": self.host,
                 },
                 {
                     "name": "rest_url",
-                    "value": f'{PROTO}{os.getenv("MAIN_HOST")}/rest',
+                    "value": f"{self.host}/rest",
                 },
                 {
                     "name": "rest_token",
                     "value": str(token),
                 },
                 {
                     "name": "tus_url",
-                    "value": f'{PROTO}{os.getenv("MAIN_HOST")}/files/',
+                    "value": f"{self.host}/files/",
                 },
                 {
                     "name": "project_id",
@@ -394,6 +396,21 @@ def start_algorithm(
             "name": self.alg.name,
         }
 
+        # Set any steps in the templates to disable eviction
+        for tidx in range(len(manifest["spec"]["templates"])):
+            if "container" in manifest["spec"]["templates"][tidx]:
+                metadata = manifest["spec"]["templates"][tidx].get("metadata", {})
+                annotations = metadata.get("annotations", {})
+                annotations = {
+                    "cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
+                    **annotations,
+                }
+                metadata = {
+                    **metadata,
+                    "annotations": annotations,
+                }
+                manifest["spec"]["templates"][tidx]["metadata"] = metadata
+
         # Set exit handler that sends an email if email specs are given
         if success_email_spec is not None or failure_email_spec is not None:
             manifest["spec"]["onExit"] = "exit-handler"
@@ -422,7 +439,7 @@ def start_algorithm(
                                 f"Authorization: Token {token}",
                                 "-d",
                                 json.dumps(success_email_spec),
-                                f'{PROTO}{os.getenv("MAIN_HOST")}/rest/Email/{project}',
+                                f"{self.host}/rest/Email/{project}",
                             ],
                         },
                     }
@@ -450,7 +467,7 @@ def start_algorithm(
                                 f"Authorization: Token {token}",
                                 "-d",
                                 json.dumps(failure_email_spec),
-                                f'{PROTO}{os.getenv("MAIN_HOST")}/rest/Email/{project}',
+                                f"{self.host}/rest/Email/{project}",
                             ],
                         },
                     }