Skip to content
This repository has been archived by the owner on Aug 9, 2021. It is now read-only.

Commit

Permalink
Fix visits (#332)
Browse files Browse the repository at this point in the history 
* unidentified in the future were not grouping with past tags correctly
  • Loading branch information
GP authored Nov 27, 2020
1 parent 19ec5eb commit e864612
Show file tree
Hide file tree
Showing 3 changed files with 181 additions and 104 deletions.
140 changes: 128 additions & 12 deletions api/V1/Visits.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,94 @@ function getTrackTag(trackTags: TrackTag[], userID: number): TrackTag | null {
}
}

class DeviceSummary {
deviceMap: DeviceVisitMap;
lastRecTime: Moment;
constructor() {
this.deviceMap = {};
}
generateVisits(
rec: any,
queryOffset: number,
complete: boolean = false,
userId: number
) {
this.lastRecTime = moment(rec.recordingDateTime);
let devVisits = this.deviceMap[rec.DeviceId];
if (!devVisits) {
devVisits = new DeviceVisits(
rec.Device.devicename,
rec.Group.groupname,
rec.DeviceId,
userId
);
this.deviceMap[rec.DeviceId] = devVisits;
}
devVisits.calculateNewVisits(rec, queryOffset, complete);
}
earliestIncompleteOffset(): number | null {
var offset = null;
for (const device of Object.values(this.deviceMap)) {
for (const visit of device.visits) {
if (!visit.incomplete) {
break;
}
if (offset == null) {
offset = visit.queryOffset;
} else {
offset = Math.min(offset, visit.queryOffset);
}
}
}
return offset;
}
checkForCompleteVisits() {
for (const device of Object.values(this.deviceMap)) {
device.checkForCompleteVisits(this.lastRecTime);
}
}
markCompleted() {
var visits = 0;
for (const device of Object.values(this.deviceMap)) {
for (const visit of device.visits) {
if (!visit.incomplete) {
break;
}
visit.incomplete = false;
}
}
}
completeVisitsCount(): number {
var visits = 0;
for (const device of Object.values(this.deviceMap)) {
visits += device.visits.filter((v) => !v.incomplete).length;
}
return visits;
}
removeIncompleteVisits() {
for (const device of Object.values(this.deviceMap)) {
device.removeIncompleteVisits();
if (device.visits.length == 0) {
delete this.deviceMap[device];
}
}
}
audiFileIds(): Set<number> {
const audioFileIds: Set<number> = new Set();

for (const device of Object.values(this.deviceMap)) {
device.audioFileIds.forEach((id) => audioFileIds.add(id));
}
return audioFileIds;
}
completeVisits(): Visit[] {
var visits: Visit[] = [];
for (const device of Object.values(this.deviceMap)) {
visits.push(...device.visits.filter((v) => !v.incomplete));
}
return visits;
}
}
class DeviceVisits {
animals: AnimalMap;
firstVisit: Visit | null;
Expand All @@ -61,6 +149,7 @@ class DeviceVisits {
visitCount: number;
eventCount: number;
audioBait: boolean;
visits: Visit[];
constructor(
public deviceName: string,
public groupName: string,
Expand All @@ -73,6 +162,15 @@ class DeviceVisits {
this.visitCount = 0;
this.eventCount = 0;
this.audioBait = false;
this.visits = [];
}
checkForCompleteVisits(lastRecTime: Moment) {
for (const visit of this.visits.reverse()) {
if (!visit.incomplete) {
break;
}
visit.incomplete = isWithinVisitInterval(visit.start, lastRecTime);
}
}

removeIncompleteVisits() {
Expand Down Expand Up @@ -113,10 +211,11 @@ class DeviceVisits {
queryOffset: number,
complete: boolean = false
): Visit[] {
const visits: Visit[] = [];
const visits = [];

const tracks = rec.Tracks;
this.sortTracks(tracks);

for (const track of tracks) {
const event = this.calculateTrackTagEvent(rec, track);
if (event == null) {
Expand All @@ -130,13 +229,24 @@ class DeviceVisits {
visits.push(newVisit);
}
}
this.visits.push(...visits);
return visits;
}

sortTracks(tracks: Track[]) {
tracks.sort(function (a, b) {
if (a.data && a.data.start_s && a.data.end_s) {
return b.data.start_s - a.data.start_s || b.id - a.id;
if (
a.data &&
b.data &&
a.data.start_s != undefined &&
b.data.start_s != undefined
) {
const res = b.data.start_s - a.data.start_s;
if (res == 0) {
return b.id - a.id;
} else {
return res;
}
} else {
return 0;
}
Expand Down Expand Up @@ -210,31 +320,30 @@ class DeviceVisits {
this.addAudioFileIds(newItem);

if (newItem instanceof Visit) {
this.firstVisit = newItem as Visit;
if (tag.what != unidentified) {
this.recheckUnidentified(this.firstVisit);
this.mergePreviousVisit(newItem as Visit);
}
this.firstVisit = newItem as Visit;
}
return newItem;
}

// as we get new visits previous unidentified events may need to be added to this visit
recheckUnidentified(visit: Visit) {
mergePreviousVisit(visit: Visit) {
const unVisit = this.firstVisit;

if (unVisit && unVisit.what == unidentified) {
let unEvent = unVisit.events[unVisit.events.length - 1];
let insertIndex = visit.events.length;
while (unEvent && visit.isPartOfVisit(unEvent.end)) {
while (unEvent && isWithinVisitInterval(visit.end, unEvent.start)) {
unEvent.wasUnidentified = true;
visit.addEventAtIndex(unEvent, insertIndex);
visit.addEventAtIndex(unEvent, 0);
unVisit.removeEventAtIndex(unVisit.events.length - 1);
unEvent = unVisit.events[unVisit.events.length - 1];
}

if (unVisit.events.length == 0) {
const unVisitSummary = this.animals[unidentified];
unVisitSummary.removeVisitAtIndex(0);
unVisitSummary.visits.pop();
this.visits.pop();
if (unVisitSummary.visits.length == 0) {
delete this.animals[unidentified];
}
Expand Down Expand Up @@ -533,4 +642,11 @@ export interface DeviceVisitMap {
export default function () {
console.log("");
}
export { DeviceVisits, VisitSummary, Visit, VisitEvent, TrackStartEnd };
export {
DeviceSummary,
DeviceVisits,
VisitSummary,
Visit,
VisitEvent,
TrackStartEnd
};
114 changes: 26 additions & 88 deletions api/V1/recordingUtil.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import {
VisitEvent,
DeviceVisitMap,
Visit,
DeviceSummary,
isWithinVisitInterval
} from "./Visits";

Expand Down Expand Up @@ -80,9 +81,9 @@ function makeUploadHandler(mungeData?: (any) => any) {
if (data.metadata) {
await tracksFromMeta(recording, data.metadata);
}
if (data.processingState){
if (data.processingState) {
recording.processingState = data.processingState;
}else{
} else {
recording.processingState = models.Recording.uploadedState(
data.type as RecordingType
);
Expand Down Expand Up @@ -582,44 +583,6 @@ async function updateMetadata(recording: any, metadata: any) {
throw new Error("recordingUtil.updateMetadata is unimplemented!");
}

// generates new visits and returns a tuple of completeVisits and incompleteVisits
function generateVisits(
deviceMap: DeviceVisitMap,
recordings: any[],
filterOptions,
queryOffset: number,
userId: number,
gotAllRecordings: boolean
): [Visit[], Visit[]] {
let visits: Visit[] = [];
let incompleteVisits: Visit[] = [];
for (const [i, rec] of recordings.entries()) {
rec.filterData(filterOptions);

let devVisits = deviceMap[rec.DeviceId];
if (!devVisits) {
devVisits = new DeviceVisits(
rec.Device.devicename,
rec.Group.groupname,
rec.DeviceId,
userId
);
deviceMap[rec.DeviceId] = devVisits;
}
const newVisits = devVisits.calculateNewVisits(
rec,
queryOffset + i,
gotAllRecordings
);
if (gotAllRecordings) {
visits.push(...newVisits);
} else {
incompleteVisits.push(...newVisits);
}
}

return [visits, incompleteVisits];
}
// Returns a promise for the recordings visits query specified in the
// request.
async function queryVisits(
Expand Down Expand Up @@ -659,16 +622,13 @@ async function queryVisits(
'"Recording"."recordingDateTime" + interval \'1 day\''
);

const audioFileIds: Set<number> = new Set();
const deviceMap: DeviceVisitMap = {};
let visits: Visit[] = [];
const devSummary = new DeviceSummary();
const filterOptions = models.Recording.makeFilterOptions(
request.user,
request.filterOptions
);
let numRecordings = 0;
let remainingVisits = requestVisits;
let incompleteVisits: Visit[] = [];
let totalCount, recordings, gotAllRecordings;

while (gotAllRecordings || remainingVisits > 0) {
Expand All @@ -685,67 +645,45 @@ async function queryVisits(
if (recordings.length == 0) {
break;
}

const [newVisits, newIncomplete] = generateVisits(
deviceMap,
recordings,
filterOptions,
request.query.offset || 0,
request.user.id,
gotAllRecordings
);
visits.push(...newVisits);
incompleteVisits.push(...newIncomplete);

if (!gotAllRecordings) {
const lastRecStart = moment(
recordings[recordings.length - 1].recordingDateTime
for (const [i, rec] of recordings.entries()) {
rec.filterData(filterOptions);
devSummary.generateVisits(
rec,
request.query.offset + i || i,
gotAllRecordings,
request.user.id
);
}

incompleteVisits = checkForCompleteVisits(
visits,
incompleteVisits,
lastRecStart
);
if (!gotAllRecordings) {
devSummary.checkForCompleteVisits();
}

remainingVisits = requestVisits - visits.length;
remainingVisits = requestVisits - devSummary.completeVisitsCount();
builder.query.limit = Math.min(remainingVisits * 2, queryMax);
builder.query.offset += recordings.length;
}

let queryOffset = 0;
// mark all as complete
if (gotAllRecordings) {
incompleteVisits.forEach((elem) => {
elem.incomplete = false;
});

visits.push(...incompleteVisits);
incompleteVisits = [];
devSummary.markCompleted();
} else {
devSummary.removeIncompleteVisits();
}
const audioFileIds = devSummary.audiFileIds();

// remove incomplete visits and get all audio file ids
for (const device in deviceMap) {
const deviceVisits = deviceMap[device];
deviceVisits.audioFileIds.forEach((id) => audioFileIds.add(id));
if (!gotAllRecordings) {
deviceVisits.removeIncompleteVisits();
}
if (deviceVisits.visitCount == 0) {
delete deviceMap[device];
}
}
const visits = devSummary.completeVisits();
visits.sort(function (a, b) {
return b.start > a.start ? 1 : -1;
});

// get the offset to use for future queries
if (incompleteVisits.length > 0) {
queryOffset = incompleteVisits[0].queryOffset;
} else if (visits.length > 0) {
queryOffset = devSummary.earliestIncompleteOffset();
if (queryOffset == null && visits.length > 0) {
queryOffset = visits[visits.length - 1].queryOffset + 1;
}

visits = visits.filter((v) => !v.incomplete);

// Bulk look up file details of played audio events.
const audioFileNames = new Map();
for (const f of await models.File.getMultiple(Array.from(audioFileIds))) {
Expand All @@ -762,7 +700,7 @@ async function queryVisits(

return {
visits: visits,
rows: deviceMap,
rows: devSummary.deviceMap,
hasMoreVisits: !gotAllRecordings,
totalRecordings: totalCount,
queryOffset: queryOffset,
Expand Down
Loading

0 comments on commit e864612

Please sign in to comment.