From b4d53a71fc216d514f3b165e39ba8bcab40bde61 Mon Sep 17 00:00:00 2001 From: GP Date: Mon, 30 Sep 2024 11:44:15 +0200 Subject: [PATCH] Add event (#212) * event stuff * update tracking event and metadata info * tidy up * build travis --------- Co-authored-by: gferraro --- .travis.yml | 8 ++- pirequirements.txt | 2 +- pyproject.toml | 4 +- requirements.txt | 2 +- src/classify/trackprediction.py | 92 +++++++++++++++++++++----------- src/dbustest.py | 35 ++++++++++-- src/piclassifier/piclassifier.py | 27 +++++----- src/piclassifier/piclassify.py | 27 +++++++--- src/piclassifier/processor.py | 9 +++- src/piclassifier/service.py | 70 +++++++++++++++++++----- tests/config.toml | 3 +- 11 files changed, 207 insertions(+), 72 deletions(-) diff --git a/.travis.yml b/.travis.yml index 910518a8..f7f59ee9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,4 +47,10 @@ deploy: script: bash ./pypi_push.sh on: branch: pi-classifier - skip_cleanup: 'true' \ No newline at end of file + skip_cleanup: 'true' + - provider: script + script: bash ./pypi_push.sh + on: + branch: add-event + skip_cleanup: 'true' + \ No newline at end of file diff --git a/pirequirements.txt b/pirequirements.txt index 72fde5a9..c47dd8d7 100644 --- a/pirequirements.txt +++ b/pirequirements.txt @@ -12,7 +12,7 @@ tables==3.8.0 h5py==3.8.0 pyyaml==6.0 pillow==10.0.1 -attrs==19.2.0 +attrs==24.2.0 filelock==3.0.12 Astral==1.10.1 timezonefinder==4.1.0 diff --git a/pyproject.toml b/pyproject.toml index 001349e6..dc53c1c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "hatchling.build" [project] name = "classifier_pipeline" -version = "0.0.16" +version = "0.0.17" authors = [ { name="Giampaolo Ferraro", email="giampaolo@cacophony.org.nz" }, ] @@ -35,7 +35,7 @@ dependencies = [ "h5py==3.8.0", "pyyaml==6.0", "pillow==10.0.1", - "attrs==19.2.0", + "attrs==24.2.0", "filelock==3.0.12", "Astral==1.10.1", "timezonefinder==4.1.0", diff --git a/requirements.txt b/requirements.txt index 8458878a..38b432c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ tables~=3.8.0 h5py~=3.9.0 pyyaml>=4.2b1 pillow~=10.0.1 -attrs~=19.1 +attrs~=24.2.0 filelock~=3.0.12 Astral~=1.10.1 timezonefinder~=6.2.0 diff --git a/src/classify/trackprediction.py b/src/classify/trackprediction.py index 9fed2472..464569d5 100644 --- a/src/classify/trackprediction.py +++ b/src/classify/trackprediction.py @@ -1,6 +1,8 @@ import attr import logging import numpy as np +import time +from attrs import define, field # uniform prior stats start with uniform distribution. This is the safest bet, but means that # it takes a while to make predictions. When off the first prediction is used instead causing @@ -55,6 +57,29 @@ def classify_time(self): return np.sum(classify_time) +@define +class Prediction: + prediction = field() + smoothed_prediction = field() + frames = field() + predicted_at_frame = field() + mass = field() + predicted_time = field(init=False) + + def __attrs_post_init__(self): + self.predicted_time = time.time() + + def get_metadata(self): + meta = attr.asdict(self) + meta["smoothed_prediction"] = np.uint32(np.round(self.smoothed_prediction)) + meta["prediction"] = np.uint8(np.round(100 * self.prediction)) + return meta + + def clarity(self): + best = np.argsort(self.prediction) + return self.prediction[best[-1]] - self.prediction[best[-2]] + + class TrackPrediction: """ Class to hold the information about the predicted class of a track. @@ -69,9 +94,7 @@ def __init__(self, track_id, labels, keep_all=True, start_frame=None): fp_index = None self.track_id = track_id self.predictions = [] - self.prediction_frames = [] self.fp_index = fp_index - self.smoothed_predictions = [] self.class_best_score = None self.start_frame = start_frame @@ -87,15 +110,20 @@ def classified_clip( self, predictions, smoothed_predictions, prediction_frames, top_score=None ): self.num_frames_classified = len(predictions) - if smoothed_predictions is None: - self.smoothed_predictions = predictions - else: - self.smoothed_predictions = smoothed_predictions - self.predictions = predictions - self.prediction_frames = prediction_frames + for prediction, smoothed_prediction, frames in zip( + predictions, smoothed_predictions, prediction_frames + ): + prediction = Prediction( + prediction, + smoothed_prediction, + frames, + np.amax(frames), + None, + ) + self.predictions.append(prediction) if self.num_frames_classified > 0: - self.class_best_score = np.sum(self.smoothed_predictions, axis=0) + self.class_best_score = np.sum(smoothed_predictions, axis=0) # normalize so it sums to 1 if top_score is None: self.class_best_score = self.class_best_score / np.sum( @@ -112,36 +140,45 @@ def normalize_score(self): self.class_best_score ) - def classified_frames(self, frame_numbers, prediction, mass): + def classified_frames(self, frame_numbers, predictions, mass): self.num_frames_classified += len(frame_numbers) - smoothed_prediction = prediction**2 * mass + self.last_frame_classified = np.max(frame_numbers) + smoothed_prediction = predictions**2 * mass + prediction = Prediction( + predictions, + smoothed_prediction, + frame_numbers, + self.last_frame_classified, + mass, + ) if self.keep_all: - self.prediction_frames.append(frame_numbers) self.predictions.append(prediction) - self.smoothed_predictions.append(smoothed_prediction) - else: - self.prediction_frames = [frame_numbers] self.predictions = [prediction] - self.smoothed_predictions = [smoothed_prediction] + if self.class_best_score is None: self.class_best_score = smoothed_prediction.copy() else: self.class_best_score += smoothed_prediction - def classified_frame(self, frame_number, prediction, mass): + def classified_frame(self, frame_number, predictions, mass): self.prediction_frames.append([frame_number]) self.last_frame_classified = frame_number self.num_frames_classified += 1 self.masses.append(mass) smoothed_prediction = prediction * prediction * mass + + prediction = Prediction( + predictions, + smoothed_prediction, + frame_number, + self.last_frame_classified, + mass, + ) if self.keep_all: self.predictions.append(prediction) - self.smoothed_predictions.append(smoothed_prediction) - else: self.predictions = [prediction] - self.smoothed_predictions = [smoothed_prediction] if self.class_best_score is None: self.class_best_score = smoothed_prediction @@ -257,9 +294,7 @@ def max_score(self): return float(np.amax(self.class_best_score)) def clarity_at(self, frame): - pred = self.predictions[frame] - best = np.argsort(pred) - return pred[best[-1]] - pred[best[-2]] + return self.predictions[frame].clarity @property def clarity(self): @@ -359,13 +394,10 @@ def get_metadata(self): ) prediction_meta["clarity"] = round(self.clarity, 3) if self.clarity else 0 prediction_meta["all_class_confidences"] = {} - if self.prediction_frames is not None: - prediction_meta["prediction_frames"] = self.prediction_frames - - # for ease always multiply by 100, depending on smoothing applied values might be large - prediction_meta["predictions"] = np.uint32( - np.round(100 * self.smoothed_predictions) - ) + preds = [] + for p in self.predictions: + preds.append(p.get_metadata()) + prediction_meta["predictions"] = preds if self.class_best_score is not None: for i, value in enumerate(self.class_best_score): label = self.labels[i] diff --git a/src/dbustest.py b/src/dbustest.py index 69a4f2b8..53733684 100644 --- a/src/dbustest.py +++ b/src/dbustest.py @@ -14,8 +14,20 @@ import threading from datetime import datetime +labels = [] -def catchall_tracking_signals_handler(what, confidence, region, tracking): + +def catchall_tracking_signals_handler( + prediction, + what, + confidence, + region, + frame, + mass, + blank, + tracking, + last_prediction_frame, +): print( "Received a trackng signal and it says " + what, confidence, @@ -23,7 +35,21 @@ def catchall_tracking_signals_handler(what, confidence, region, tracking): region, " tracking?", tracking, + "prediction", + prediction, + "frame", + frame, + "mass", + mass, + "blank", + blank, + "last prediction frame", + last_prediction_frame, ) + index = 0 + for x in prediction: + print("For ", labels[index], " have confidence ", int(x), "%") + index += 1 def catchall_rec_signals_handler(dt, is_recording): @@ -49,13 +75,16 @@ def quit(self): self.loop.quit() def run_server(self): + dbus_object = None try: bus = dbus.SystemBus() - object = bus.get_object(DBUS_NAME, DBUS_PATH) + dbus_object = bus.get_object(DBUS_NAME, DBUS_PATH) except dbus.exceptions.DBusException as e: print("Failed to initialize D-Bus object: '%s'" % str(e)) sys.exit(2) - + global labels + labels = dbus_object.ClassificationLabels() + print("Labels are ", labels) bus.add_signal_receiver( self.callback, dbus_interface=DBUS_NAME, diff --git a/src/piclassifier/piclassifier.py b/src/piclassifier/piclassifier.py index 8a07d761..a875ba7c 100644 --- a/src/piclassifier/piclassifier.py +++ b/src/piclassifier/piclassifier.py @@ -82,7 +82,7 @@ def __init__( self.constant_recorder = None self._output_dir = thermal_config.recorder.output_dir self.headers = headers - super().__init__() + self.classifier = None self.frame_num = 0 self.clip = None self.enable_per_track_information = False @@ -277,6 +277,7 @@ def __init__( except ValueError: self.fp_index = None self.startup_classifier() + super().__init__() def new_clip(self, preview_frames): self.clip = Clip( @@ -438,20 +439,20 @@ def identify_last_frame(self): self.tracking = track track_prediction.normalize_score() self.service.tracking( - track_prediction.predicted_tag(), - track_prediction.max_score, - track.bounds_history[-1].to_ltrb(), + track_prediction.class_best_score, + track.bounds_history[-1], True, + track_prediction.last_frame_classified, ) elif track_prediction.tracking: track_prediction.tracking = False self.tracking = None track_prediction.normalize_score() self.service.tracking( - track_prediction.predicted_tag(), - track_prediction.max_score, - track.bounds_history[-1].to_ltrb(), + track_prediction.class_best_score, + track.bounds_history[-1], False, + track_prediction.last_frame_classified, ) new_prediction = True @@ -584,17 +585,19 @@ def process_frame(self, lepton_frame, received_at): tracking = self.tracking in self.clip.active_tracks score = 0 prediction = "" + all_scores = None + last_prediction = 0 if self.classify: track_prediction = self.predictions.prediction_for( self.tracking.get_id() ) - prediction = track_prediction.predicted_tag() - score = track_prediction.max_score + all_scores = track_prediction.class_best_score + last_prediction = track_prediction.last_frame_classified self.service.tracking( - prediction, - score, - self.tracking.bounds_history[-1].to_ltrb(), + all_scores, + self.tracking.bounds_history[-1], tracking, + last_prediction, ) if not tracking: diff --git a/src/piclassifier/piclassify.py b/src/piclassifier/piclassify.py index 8857f89b..ee381733 100644 --- a/src/piclassifier/piclassify.py +++ b/src/piclassifier/piclassify.py @@ -54,6 +54,14 @@ def parse_args(): parser.add_argument( "--ir", action="count", help="Path to pi-config file (config.toml) to use" ) + + parser.add_argument( + "--fps", + type=int, + default=None, + help="When running a file through specify the frame rate you want it to run at, otherwise it runs as fast as the cpu can", + ) + args = parser.parse_args() return args @@ -76,7 +84,9 @@ def main(): ) if args.file: - return parse_file(args.file, config, thermal_config, args.preview_type) + return parse_file( + args.file, config, thermal_config, args.preview_type, args.fps + ) process_queue = multiprocessing.Queue() @@ -159,19 +169,19 @@ def file_changed(event): os._exit(0) -def parse_file(file, config, thermal_config, preview_type): +def parse_file(file, config, thermal_config, preview_type, fps): _, ext = os.path.splitext(file) thermal_config.recorder.rec_window = rec_window = TimeWindow( RelAbsTime(""), RelAbsTime(""), None, None, 0 ) if ext == ".cptv": - parse_cptv(file, config, thermal_config.config_file, preview_type) + parse_cptv(file, config, thermal_config.config_file, preview_type, fps) else: - parse_ir(file, config, thermal_config, preview_type) + parse_ir(file, config, thermal_config, preview_type, fps) -def parse_ir(file, config, thermal_config, preview_type): +def parse_ir(file, config, thermal_config, preview_type, fps): from piclassifier import irmotiondetector import cv2 @@ -179,6 +189,8 @@ def parse_ir(file, config, thermal_config, preview_type): count = 0 vidcap = cv2.VideoCapture(file) while True: + if fps is not None: + time.sleep(1 / fps) success, image = vidcap.read() if not success: break @@ -222,7 +234,7 @@ def parse_ir(file, config, thermal_config, preview_type): pi_classifier.disconnected() -def parse_cptv(file, config, thermal_config_file, preview_type): +def parse_cptv(file, config, thermal_config_file, preview_type, fps): from cptv import CPTVReader with open(file, "rb") as f: @@ -258,7 +270,8 @@ def parse_cptv(file, config, thermal_config_file, preview_type): pi_classifier.motion_detector._background._background = frame.pix continue pi_classifier.process_frame(frame, time.time()) - + if fps is not None: + time.sleep(1.0 / fps) pi_classifier.disconnected() diff --git a/src/piclassifier/processor.py b/src/piclassifier/processor.py index 929ab5dc..8546b22b 100644 --- a/src/piclassifier/processor.py +++ b/src/piclassifier/processor.py @@ -27,7 +27,14 @@ def __init__( self, ): self.service = SnapshotService( - self.get_recent_frame, self.headers, self.take_snapshot + self.get_recent_frame, + self.headers, + self.take_snapshot, + ( + self.classifier.labels + if self.classifier is not None + else ["Not classifying"] + ), ) @abstractmethod diff --git a/src/piclassifier/service.py b/src/piclassifier/service.py index cb610752..41f8f6dc 100644 --- a/src/piclassifier/service.py +++ b/src/piclassifier/service.py @@ -17,11 +17,12 @@ class Service(dbus.service.Object): - def __init__(self, dbus, get_frame, headers, take_snapshot_fn): + def __init__(self, dbus, get_frame, headers, take_snapshot_fn, labels): super().__init__(dbus, DBUS_PATH) self.get_frame = get_frame self.headers = headers self.take_snapshot = take_snapshot_fn + self.labels = labels @dbus.service.method( DBUS_NAME, @@ -100,8 +101,24 @@ def TakeTestRecording(self): return result - @dbus.service.signal(DBUS_NAME, signature="siaib") - def Tracking(self, what, confidence, region, tracking): + @dbus.service.method(DBUS_NAME, signature="as") + def ClassificationLabels(self): + logging.info("Getting labels %s", self.labels) + return self.labels + + @dbus.service.signal(DBUS_NAME, signature="aisiaiiibbi") + def Tracking( + self, + prediction, + what, + confidence, + region, + frame, + mass, + blank, + tracking, + last_prediction_frame, + ): pass @dbus.service.signal(DBUS_NAME, signature="xb") @@ -110,13 +127,13 @@ def Recording(self, timestamp, is_recording): class SnapshotService: - def __init__(self, get_frame, headers, take_snapshot_fn): + def __init__(self, get_frame, headers, take_snapshot_fn, labels): DBusGMainLoop(set_as_default=True) dbus.mainloop.glib.threads_init() self.loop = GLib.MainLoop() self.t = threading.Thread( target=self.run_server, - args=(get_frame, headers, take_snapshot_fn), + args=(get_frame, headers, take_snapshot_fn, labels), ) self.t.start() self.service = None @@ -124,23 +141,50 @@ def __init__(self, get_frame, headers, take_snapshot_fn): def quit(self): self.loop.quit() - def run_server(self, get_frame, headers, take_snapshot_fn): + def run_server(self, get_frame, headers, take_snapshot_fn, labels): session_bus = dbus.SystemBus(mainloop=DBusGMainLoop()) name = dbus.service.BusName(DBUS_NAME, session_bus) - self.service = Service(session_bus, get_frame, headers, take_snapshot_fn) + self.service = Service( + session_bus, get_frame, headers, take_snapshot_fn, labels + ) self.loop.run() - def tracking(self, what, confidence, region, tracking): + def tracking(self, prediction, region, tracking, last_prediction_frame): logging.debug( - "Tracking %s animal %s confidence %s at %s", - what, - round(100 * confidence), - region, + "Tracking? %s region %s prediction %s", tracking, + region, + prediction, ) if self.service is None: return - self.service.Tracking(what, round(100 * confidence), region, tracking) + if prediction is not None: + predictions = prediction.copy() + predictions = np.uint8(np.round(predictions * 100)) + best = np.argmax(predictions) + self.service.Tracking( + predictions, + self.service.labels[best], + predictions[best], + region.to_ltrb(), + region.frame_number, + region.mass, + region.blank, + tracking, + last_prediction_frame, + ) + else: + self.service.Tracking( + [], + "", + 0, + region.to_ltrb(), + region.frame_number, + region.mass, + region.blank, + tracking, + last_prediction_frame, + ) def recording(self, is_recording): if self.service is None: diff --git a/tests/config.toml b/tests/config.toml index 8c194fe3..0accdf69 100644 --- a/tests/config.toml +++ b/tests/config.toml @@ -15,8 +15,9 @@ managementd = 80 [thermal-motion] + tracking-events = true do-tracking = true - run-classifier = true + run-classifier = true count-thresh = 3 delta-thresh = 50 edge-pixels = 1