From 445315e9e364ac6fba40b283fbbfab6303fe548d Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Tue, 18 Apr 2023 11:09:48 +0300 Subject: [PATCH 1/8] Add Deepstack detector plugin with configurable API URL, timeout, and API key --- frigate/detectors/plugins/deepstack.py | 86 ++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 frigate/detectors/plugins/deepstack.py diff --git a/frigate/detectors/plugins/deepstack.py b/frigate/detectors/plugins/deepstack.py new file mode 100644 index 0000000000..cbc8af44af --- /dev/null +++ b/frigate/detectors/plugins/deepstack.py @@ -0,0 +1,86 @@ +import logging +import numpy as np +import requests +import io + +from frigate.detectors.detection_api import DetectionApi +from frigate.detectors.detector_config import BaseDetectorConfig +from typing import Literal +from pydantic import Extra, Field +from PIL import Image + + +logger = logging.getLogger(__name__) + +DETECTOR_KEY = "deepstack" + + +class DeepstackDetectorConfig(BaseDetectorConfig): + type: Literal[DETECTOR_KEY] + api_url: str = Field(default="http://localhost:80/v1/vision/detection", title="DeepStack API URL") + api_timeout: float = Field(default=0.1, title="DeepStack API timeout (in seconds)") + api_key: str = Field(default="", title="DeepStack API key (if required)") + +class DeepStack(DetectionApi): + type_key = DETECTOR_KEY + + def __init__(self, detector_config: DeepstackDetectorConfig): + self.api_url = detector_config.api_url + self.api_timeout = detector_config.api_timeout + self.api_key = detector_config.api_key + self.labels = self.load_labels("/labelmap.txt") + + self.h = detector_config.model.height + self.w = detector_config.model.width + + def load_labels(self, path, encoding="utf-8"): + """Loads labels from file (with or without index numbers). + Args: + path: path to label file. + encoding: label file encoding. + Returns: + Dictionary mapping indices to labels. + """ + with open(path, "r", encoding=encoding) as f: + labels = {index: "unknown" for index in range(91)} + lines = f.readlines() + if not lines: + return {} + + if lines[0].split(" ", maxsplit=1)[0].isdigit(): + pairs = [line.split(" ", maxsplit=1) for line in lines] + labels.update({int(index): label.strip() for index, label in pairs}) + else: + labels.update({index: line.strip() for index, line in enumerate(lines)}) + return labels + + def get_label_index(self, label_value): + for index, value in self.labels.items(): + if value == label_value: + return index + return None + + def detect_raw(self, tensor_input): + image_data = np.squeeze(tensor_input).astype(np.uint8) + image = Image.fromarray(image_data) + with io.BytesIO() as output: + image.save(output, format="JPEG") + image_bytes = output.getvalue() + data = {"api_key": self.api_key} + response = requests.post(self.api_url, files={"image": image_bytes}, timeout=self.api_timeout) + response_json = response.json() + detections = np.zeros((20, 6), np.float32) + + for i, detection in enumerate(response_json["predictions"]): + if detection["confidence"] < 0.4: + break + detections[i] = [ + int(self.get_label_index(detection["label"])), + float(detection["confidence"]), + detection["y_min"] / self.h, + detection["x_min"] / self.w, + detection["y_max"] / self.h, + detection["x_max"] / self.w, + ] + + return detections From f02a9d62386e4043a7c28244d03fde40d162d24d Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Sun, 23 Apr 2023 17:21:45 +0300 Subject: [PATCH 2/8] Update DeepStack plugin to recognize 'truck' as 'car' for label indexing --- frigate/detectors/plugins/deepstack.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/frigate/detectors/plugins/deepstack.py b/frigate/detectors/plugins/deepstack.py index cbc8af44af..6cd5be5905 100644 --- a/frigate/detectors/plugins/deepstack.py +++ b/frigate/detectors/plugins/deepstack.py @@ -55,10 +55,12 @@ def load_labels(self, path, encoding="utf-8"): return labels def get_label_index(self, label_value): + if label_value.lower() == 'truck': + label_value = 'car' for index, value in self.labels.items(): - if value == label_value: + if value == label_value.lower(): return index - return None + return -1 def detect_raw(self, tensor_input): image_data = np.squeeze(tensor_input).astype(np.uint8) @@ -74,8 +76,11 @@ def detect_raw(self, tensor_input): for i, detection in enumerate(response_json["predictions"]): if detection["confidence"] < 0.4: break + label = self.get_label_index(detection["label"]) + if label < 0: + break detections[i] = [ - int(self.get_label_index(detection["label"])), + label, float(detection["confidence"]), detection["y_min"] / self.h, detection["x_min"] / self.w, From 2794cd0593565b0fbc331c2e794917e9ef500c56 Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Sun, 23 Apr 2023 18:31:13 +0300 Subject: [PATCH 3/8] Add debug logging to DeepStack plugin for better monitoring and troubleshooting --- frigate/detectors/plugins/deepstack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frigate/detectors/plugins/deepstack.py b/frigate/detectors/plugins/deepstack.py index 6cd5be5905..f1b25492e5 100644 --- a/frigate/detectors/plugins/deepstack.py +++ b/frigate/detectors/plugins/deepstack.py @@ -74,10 +74,13 @@ def detect_raw(self, tensor_input): detections = np.zeros((20, 6), np.float32) for i, detection in enumerate(response_json["predictions"]): + logger.debug(f"Response: {detection}") if detection["confidence"] < 0.4: + logger.debug(f"Break due to confidence < 0.4") break label = self.get_label_index(detection["label"]) if label < 0: + logger.debug(f"Break due to unknown label") break detections[i] = [ label, From 1a2f8c568e67f1c3899c07b44bb121b459508c2f Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Sun, 23 Apr 2023 18:40:04 +0300 Subject: [PATCH 4/8] Refactor DeepStack label loading from file to use merged labelmap --- frigate/detectors/plugins/deepstack.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/frigate/detectors/plugins/deepstack.py b/frigate/detectors/plugins/deepstack.py index f1b25492e5..88b47a1dbf 100644 --- a/frigate/detectors/plugins/deepstack.py +++ b/frigate/detectors/plugins/deepstack.py @@ -28,31 +28,10 @@ def __init__(self, detector_config: DeepstackDetectorConfig): self.api_url = detector_config.api_url self.api_timeout = detector_config.api_timeout self.api_key = detector_config.api_key - self.labels = self.load_labels("/labelmap.txt") + self.labels = detector_config.model.merged_labelmap self.h = detector_config.model.height self.w = detector_config.model.width - - def load_labels(self, path, encoding="utf-8"): - """Loads labels from file (with or without index numbers). - Args: - path: path to label file. - encoding: label file encoding. - Returns: - Dictionary mapping indices to labels. - """ - with open(path, "r", encoding=encoding) as f: - labels = {index: "unknown" for index in range(91)} - lines = f.readlines() - if not lines: - return {} - - if lines[0].split(" ", maxsplit=1)[0].isdigit(): - pairs = [line.split(" ", maxsplit=1) for line in lines] - labels.update({int(index): label.strip() for index, label in pairs}) - else: - labels.update({index: line.strip() for index, line in enumerate(lines)}) - return labels def get_label_index(self, label_value): if label_value.lower() == 'truck': From aa6d99a67e2f76e6fa8b56e26ff0090af195a8d5 Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Sun, 23 Apr 2023 18:44:07 +0300 Subject: [PATCH 5/8] Black format --- frigate/detectors/plugins/deepstack.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/frigate/detectors/plugins/deepstack.py b/frigate/detectors/plugins/deepstack.py index 88b47a1dbf..716065903c 100644 --- a/frigate/detectors/plugins/deepstack.py +++ b/frigate/detectors/plugins/deepstack.py @@ -17,10 +17,13 @@ class DeepstackDetectorConfig(BaseDetectorConfig): type: Literal[DETECTOR_KEY] - api_url: str = Field(default="http://localhost:80/v1/vision/detection", title="DeepStack API URL") + api_url: str = Field( + default="http://localhost:80/v1/vision/detection", title="DeepStack API URL" + ) api_timeout: float = Field(default=0.1, title="DeepStack API timeout (in seconds)") api_key: str = Field(default="", title="DeepStack API key (if required)") + class DeepStack(DetectionApi): type_key = DETECTOR_KEY @@ -32,15 +35,15 @@ def __init__(self, detector_config: DeepstackDetectorConfig): self.h = detector_config.model.height self.w = detector_config.model.width - + def get_label_index(self, label_value): - if label_value.lower() == 'truck': - label_value = 'car' + if label_value.lower() == "truck": + label_value = "car" for index, value in self.labels.items(): if value == label_value.lower(): return index return -1 - + def detect_raw(self, tensor_input): image_data = np.squeeze(tensor_input).astype(np.uint8) image = Image.fromarray(image_data) @@ -48,10 +51,12 @@ def detect_raw(self, tensor_input): image.save(output, format="JPEG") image_bytes = output.getvalue() data = {"api_key": self.api_key} - response = requests.post(self.api_url, files={"image": image_bytes}, timeout=self.api_timeout) + response = requests.post( + self.api_url, files={"image": image_bytes}, timeout=self.api_timeout + ) response_json = response.json() detections = np.zeros((20, 6), np.float32) - + for i, detection in enumerate(response_json["predictions"]): logger.debug(f"Response: {detection}") if detection["confidence"] < 0.4: From f7f70cefaec40b425a81fa2925f237d772cd53e6 Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Tue, 2 May 2023 23:18:21 +0300 Subject: [PATCH 6/8] add documentation draft --- docs/docs/configuration/detectors.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/docs/configuration/detectors.md b/docs/docs/configuration/detectors.md index b7f442c313..8b009ee3cb 100644 --- a/docs/docs/configuration/detectors.md +++ b/docs/docs/configuration/detectors.md @@ -256,3 +256,24 @@ model: width: 416 height: 416 ``` + + +## Deepstack / CodeProject AI.Server Detector +The Deepstack/CodeProject AI.Server detector plugin for Frigate allows you to integrate Deeppstack or CodeProject.AI object detection capabilities into Frigate video surveillance system. CodeProject.AI and DeepStack is an open-source AI platforms that can be run on various devices, such as Raspberry Pi, Nvidia Jetson, and other compatible hardware. It is important to note that the integration is performed over the network, so the inference times may not be as fast as native Frigate detectors, but it still provides an efficient and reliable solution for object detection and tracking. + +### Setup +To get started with CodeProject.AI, visit their official website at https://www.codeproject.ai and follow the instructions to download and install the AI server on your preferred device. Detailed setup instructions for CodeProject.AI are outside the scope of the Frigate documentation. + +To integrate CodeProject.AI with Frigate, you'll need to make the following changes to your Frigate configuration file: + +```yaml +detectors: + deepstack: + api_url: http://:/v1/vision/detection + type: deepstack + api_timeout: 0.1 # seconds +``` + +Replace `` and `` with the IP address and port of your CodeProject.AI server. + +To verify that the integration is working correctly, start Frigate and observe the logs for any error messages related to CodeProject.AI. Additionally, you can check the Frigate web interface to see if the objects detected by CodeProject.AI are being displayed and tracked properly. \ No newline at end of file From a3af809c7e1a071d4a2c0ae3384a5760556c2321 Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Tue, 2 May 2023 23:46:39 +0300 Subject: [PATCH 7/8] fix link to codeproject website --- docs/docs/configuration/detectors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/configuration/detectors.md b/docs/docs/configuration/detectors.md index 8b009ee3cb..133395f18f 100644 --- a/docs/docs/configuration/detectors.md +++ b/docs/docs/configuration/detectors.md @@ -262,7 +262,7 @@ model: The Deepstack/CodeProject AI.Server detector plugin for Frigate allows you to integrate Deeppstack or CodeProject.AI object detection capabilities into Frigate video surveillance system. CodeProject.AI and DeepStack is an open-source AI platforms that can be run on various devices, such as Raspberry Pi, Nvidia Jetson, and other compatible hardware. It is important to note that the integration is performed over the network, so the inference times may not be as fast as native Frigate detectors, but it still provides an efficient and reliable solution for object detection and tracking. ### Setup -To get started with CodeProject.AI, visit their official website at https://www.codeproject.ai and follow the instructions to download and install the AI server on your preferred device. Detailed setup instructions for CodeProject.AI are outside the scope of the Frigate documentation. +To get started with CodeProject.AI, visit their official website at [www.codeproject.com](https://www.codeproject.com/Articles/5322557/CodeProject-AI-Server-AI-the-easy-way) and follow the instructions to download and install the AI server on your preferred device. Detailed setup instructions for CodeProject.AI are outside the scope of the Frigate documentation. To integrate CodeProject.AI with Frigate, you'll need to make the following changes to your Frigate configuration file: From 11f3e3b05b3ded4b9d9a523b50249c7a6e636b4f Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Tue, 2 May 2023 23:58:44 +0300 Subject: [PATCH 8/8] Apply suggestions from code review Co-authored-by: Nicolas Mowen --- docs/docs/configuration/detectors.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/docs/configuration/detectors.md b/docs/docs/configuration/detectors.md index 133395f18f..7fbd03f797 100644 --- a/docs/docs/configuration/detectors.md +++ b/docs/docs/configuration/detectors.md @@ -257,14 +257,15 @@ model: height: 416 ``` +## Deepstack / CodeProject.AI Server Detector -## Deepstack / CodeProject AI.Server Detector -The Deepstack/CodeProject AI.Server detector plugin for Frigate allows you to integrate Deeppstack or CodeProject.AI object detection capabilities into Frigate video surveillance system. CodeProject.AI and DeepStack is an open-source AI platforms that can be run on various devices, such as Raspberry Pi, Nvidia Jetson, and other compatible hardware. It is important to note that the integration is performed over the network, so the inference times may not be as fast as native Frigate detectors, but it still provides an efficient and reliable solution for object detection and tracking. +The Deepstack / CodeProject.AI Server detector for Frigate allows you to integrate Deepstack and CodeProject.AI object detection capabilities into Frigate. CodeProject.AI and DeepStack are open-source AI platforms that can be run on various devices such as the Raspberry Pi, Nvidia Jetson, and other compatible hardware. It is important to note that the integration is performed over the network, so the inference times may not be as fast as native Frigate detectors, but it still provides an efficient and reliable solution for object detection and tracking. ### Setup -To get started with CodeProject.AI, visit their official website at [www.codeproject.com](https://www.codeproject.com/Articles/5322557/CodeProject-AI-Server-AI-the-easy-way) and follow the instructions to download and install the AI server on your preferred device. Detailed setup instructions for CodeProject.AI are outside the scope of the Frigate documentation. -To integrate CodeProject.AI with Frigate, you'll need to make the following changes to your Frigate configuration file: +To get started with CodeProject.AI, visit their [official website](https://www.codeproject.com/Articles/5322557/CodeProject-AI-Server-AI-the-easy-way) to follow the instructions to download and install the AI server on your preferred device. Detailed setup instructions for CodeProject.AI are outside the scope of the Frigate documentation. + +To integrate CodeProject.AI into Frigate, you'll need to make the following changes to your Frigate configuration file: ```yaml detectors: