From a455a3a288be48077585b31af0e7356c6cbb3a38 Mon Sep 17 00:00:00 2001 From: GP Date: Mon, 22 Feb 2021 14:57:28 +1300 Subject: [PATCH] Add processing fields (#350) * added processing attributes --- api/V1/Group.ts | 113 +++++++++--------- api/V1/recordingUtil.ts | 31 +++-- api/middleware.ts | 15 ++- migrations/20210120160000-create-stations.js | 24 +++- models/Group.ts | 119 ++++++++++++------- models/Recording.ts | 11 +- models/Station.ts | 4 +- models/User.ts | 4 +- models/index.ts | 2 +- 9 files changed, 201 insertions(+), 122 deletions(-) diff --git a/api/V1/Group.ts b/api/V1/Group.ts index 54c4c725..23f4d802 100644 --- a/api/V1/Group.ts +++ b/api/V1/Group.ts @@ -20,34 +20,37 @@ import middleware from "../middleware"; import auth from "../auth"; import models from "../../models"; import responseUtil from "./responseUtil"; -import {body, param, query} from "express-validator/check"; +import { body, param, query } from "express-validator/check"; import { Application } from "express"; import { Validator } from "jsonschema"; const JsonSchema = new Validator(); const validateStationsJson = (val, { req }) => { - const stations = JSON.parse(val); + const stations = JSON.parse(val); - // Validate json schema of input: - JsonSchema.validate(stations, { - type: "array", - minItems: 1, - uniqueItems: true, - items: { - properties: { - name: { type: "string" }, - lat: { type: "number" }, - lng: { type: "number" }, - }, - required: ["name", "lat", "lng"] - } - }, {throwFirst: true}); - req.body.stations = stations; - return true; + // Validate json schema of input: + JsonSchema.validate( + stations, + { + type: "array", + minItems: 1, + uniqueItems: true, + items: { + properties: { + name: { type: "string" }, + lat: { type: "number" }, + lng: { type: "number" } + }, + required: ["name", "lat", "lng"] + } + }, + { throwFirst: true } + ); + req.body.stations = stations; + return true; }; - export default function (app: Application, baseUrl: string) { const apiUrl = `${baseUrl}/groups`; @@ -191,33 +194,30 @@ export default function (app: Application, baseUrl: string) { ); /** - * @api {post} /api/v1/groups/{groupIdOrName}/stations Add, Update and retire current stations belonging to group - * @apiName PostStationsForGroup - * @apiGroup Group - * @apiDescription A group admin or an admin with globalWrite permissions can update stations for a group. - * - * @apiUse V1UserAuthorizationHeader - * - * @apiParam {Number|String} group name or group id - * @apiParam {JSON} Json array of {name: string, lat: number, lng: number} - * - * @apiUse V1ResponseSuccess - * @apiUse V1ResponseError - */ + * @api {post} /api/v1/groups/{groupIdOrName}/stations Add, Update and retire current stations belonging to group + * @apiName PostStationsForGroup + * @apiGroup Group + * @apiDescription A group admin or an admin with globalWrite permissions can update stations for a group. + * + * @apiUse V1UserAuthorizationHeader + * + * @apiParam {Number|String} group name or group id + * @apiParam {JSON} Json array of {name: string, lat: number, lng: number} + * + * @apiUse V1ResponseSuccess + * @apiUse V1ResponseError + */ app.post( `${apiUrl}/:groupIdOrName/stations`, [ auth.authenticateUser, middleware.getGroupByNameOrIdDynamic(param, "groupIdOrName"), body("stations") - .exists() - .isJSON() - .withMessage("Expected JSON array") - .custom(validateStationsJson), - body("fromDate") - .isISO8601() - .toDate() - .optional(), + .exists() + .isJSON() + .withMessage("Expected JSON array") + .custom(validateStationsJson), + body("fromDate").isISO8601().toDate().optional() ], middleware.requestWrapper(async (request, response) => { const stationIds = await models.Group.addStationsToGroup( @@ -235,19 +235,19 @@ export default function (app: Application, baseUrl: string) { ); /** - * @api {get} /api/v1/groups/{groupIdOrName}/stations Retrieves all stations from a group, including retired ones. - * @apiName GetStationsForGroup - * @apiGroup Group - * @apiDescription A group member or an admin member with globalRead permissions can view stations that belong - * to a group. - * - * @apiUse V1UserAuthorizationHeader - * - * @apiParam {Number|String} group name or group id - * - * @apiUse V1ResponseSuccess - * @apiUse V1ResponseError - */ + * @api {get} /api/v1/groups/{groupIdOrName}/stations Retrieves all stations from a group, including retired ones. + * @apiName GetStationsForGroup + * @apiGroup Group + * @apiDescription A group member or an admin member with globalRead permissions can view stations that belong + * to a group. + * + * @apiUse V1UserAuthorizationHeader + * + * @apiParam {Number|String} group name or group id + * + * @apiUse V1ResponseSuccess + * @apiUse V1ResponseError + */ app.get( `${apiUrl}/:groupIdOrName/stations`, [ @@ -255,7 +255,10 @@ export default function (app: Application, baseUrl: string) { middleware.getGroupByNameOrIdDynamic(param, "groupIdOrName") ], middleware.requestWrapper(async (request, response) => { - if (request.user.hasGlobalRead() || await request.user.isInGroup(request.body.group.id)) { + if ( + request.user.hasGlobalRead() || + (await request.user.isInGroup(request.body.group.id)) + ) { const stations = await request.body.group.getStations(); return responseUtil.send(response, { statusCode: 200, @@ -265,7 +268,7 @@ export default function (app: Application, baseUrl: string) { } else { return responseUtil.send(response, { statusCode: 403, - messages: ["User is not member of group, can't list stations"], + messages: ["User is not member of group, can't list stations"] }); } }) diff --git a/api/V1/recordingUtil.ts b/api/V1/recordingUtil.ts index 4153e12d..591259fb 100644 --- a/api/V1/recordingUtil.ts +++ b/api/V1/recordingUtil.ts @@ -46,7 +46,7 @@ import { VisitSummary, isWithinVisitInterval } from "./Visits"; -import {Station, StationId} from "../../models/Station"; +import { Station, StationId } from "../../models/Station"; export interface RecordingQuery { user: User; @@ -66,9 +66,13 @@ export interface RecordingQuery { export const MIN_STATION_SEPARATION_METERS = 60; // The radius of the station is half the max distance between stations: any recording inside the radius can // be considered to belong to that station. -export const MAX_DISTANCE_FROM_STATION_FOR_RECORDING = MIN_STATION_SEPARATION_METERS / 2; +export const MAX_DISTANCE_FROM_STATION_FOR_RECORDING = + MIN_STATION_SEPARATION_METERS / 2; -export function latLngApproxDistance(a: [number, number], b: [number, number]): number { +export function latLngApproxDistance( + a: [number, number], + b: [number, number] +): number { const R = 6371e3; // Using 'spherical law of cosines' from https://www.movable-type.co.uk/scripts/latlong.html const lat1 = (a[0] * Math.PI) / 180; @@ -76,11 +80,16 @@ export function latLngApproxDistance(a: [number, number], b: [number, number]): const sinLat1 = Math.sin(lat1); const lat2 = (b[0] * Math.PI) / 180; const deltaLng = ((b[1] - a[1]) * Math.PI) / 180; - const part1 = Math.acos(sinLat1 * Math.sin(lat2) + costLat1 * Math.cos(lat2) * Math.cos(deltaLng)); + const part1 = Math.acos( + sinLat1 * Math.sin(lat2) + costLat1 * Math.cos(lat2) * Math.cos(deltaLng) + ); return part1 * R; } -export async function tryToMatchRecordingToStation(recording: Recording, stations?: Station[]): Promise { +export async function tryToMatchRecordingToStation( + recording: Recording, + stations?: Station[] +): Promise { // If the recording does not yet have a location, return if (!recording.location) { return null; @@ -94,10 +103,16 @@ export async function tryToMatchRecordingToStation(recording: Recording, station const stationDistances = []; for (const station of stations) { // See if any stations match: Looking at the location distance between this recording and the stations. - const distanceToStation = latLngApproxDistance(station.location.coordinates, recording.location.coordinates); - stationDistances.push({distanceToStation, station}); + const distanceToStation = latLngApproxDistance( + station.location.coordinates, + recording.location.coordinates + ); + stationDistances.push({ distanceToStation, station }); } - const validStationDistances = stationDistances.filter(({distanceToStation}) => distanceToStation <= MAX_DISTANCE_FROM_STATION_FOR_RECORDING); + const validStationDistances = stationDistances.filter( + ({ distanceToStation }) => + distanceToStation <= MAX_DISTANCE_FROM_STATION_FOR_RECORDING + ); // There shouldn't really ever be more than one station within our threshold distance, // since we check that stations aren't too close together when we add them. However, on the off diff --git a/api/middleware.ts b/api/middleware.ts index eb91b761..bcf5f79b 100644 --- a/api/middleware.ts +++ b/api/middleware.ts @@ -140,13 +140,16 @@ function getGroupByNameOrId(checkFunc: ValidationChainBuilder): RequestHandler { ); } -function getGroupByNameOrIdDynamic(checkFunc: ValidationChainBuilder, fieldName: string): RequestHandler { +function getGroupByNameOrIdDynamic( + checkFunc: ValidationChainBuilder, + fieldName: string +): RequestHandler { return oneOf( - [ - getModelById(models.Group, fieldName, checkFunc), - getModelByName(models.Group, fieldName, checkFunc) - ], - "Group doesn't exist or hasn't been specified." + [ + getModelById(models.Group, fieldName, checkFunc), + getModelByName(models.Group, fieldName, checkFunc) + ], + "Group doesn't exist or hasn't been specified." ); } diff --git a/migrations/20210120160000-create-stations.js b/migrations/20210120160000-create-stations.js index f8ad1289..30d4239c 100644 --- a/migrations/20210120160000-create-stations.js +++ b/migrations/20210120160000-create-stations.js @@ -19,9 +19,21 @@ module.exports = { type: Sequelize.GEOMETRY, allowNull: false }, - createdAt: { type: Sequelize.DATE, allowedNull: false, defaultValue: Sequelize.NOW }, - updatedAt: { type: Sequelize.DATE, allowedNull: false, defaultValue: Sequelize.NOW }, - retiredAt: { type: Sequelize.DATE, allowedNull: true, defaultValue: Sequelize.NULL } + createdAt: { + type: Sequelize.DATE, + allowedNull: false, + defaultValue: Sequelize.NOW + }, + updatedAt: { + type: Sequelize.DATE, + allowedNull: false, + defaultValue: Sequelize.NOW + }, + retiredAt: { + type: Sequelize.DATE, + allowedNull: true, + defaultValue: Sequelize.NULL + } }); await util.migrationAddBelongsTo(queryInterface, "Stations", "Users", { name: "lastUpdatedBy" @@ -35,7 +47,11 @@ module.exports = { name: "lastUpdatedBy" }); await util.migrationRemoveBelongsTo(queryInterface, "Stations", "Groups"); - await util.migrationRemoveBelongsTo(queryInterface, "Recordings", "Stations"); + await util.migrationRemoveBelongsTo( + queryInterface, + "Recordings", + "Stations" + ); await queryInterface.dropTable("Stations"); } }; diff --git a/models/Group.ts b/models/Group.ts index 77799117..3dd5a9a4 100644 --- a/models/Group.ts +++ b/models/Group.ts @@ -16,21 +16,25 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -import Sequelize, {Op} from "sequelize"; -import AllModels, {ModelCommon, ModelStaticCommon} from "./index"; -import {User, UserId} from "./User"; -import {CreateStationData, Station, StationId} from "./Station"; -import {Recording} from "./Recording"; +import Sequelize, { Op } from "sequelize"; +import AllModels, { ModelCommon, ModelStaticCommon } from "./index"; +import { User, UserId } from "./User"; +import { CreateStationData, Station, StationId } from "./Station"; +import { Recording } from "./Recording"; import { latLngApproxDistance, MIN_STATION_SEPARATION_METERS, tryToMatchRecordingToStation } from "../api/V1/recordingUtil"; -import {ClientError} from "../api/customErrors"; +import { ClientError } from "../api/customErrors"; import { AuthorizationError } from "../api/customErrors"; export type GroupId = number; -const retireMissingStations = (existingStations: Station[], newStationsByName: Record, userId: UserId): Promise[] => { +const retireMissingStations = ( + existingStations: Station[], + newStationsByName: Record, + userId: UserId +): Promise[] => { const retirePromises = []; const numExisting = existingStations.length; for (let i = 0; i < numExisting; i++) { @@ -47,59 +51,76 @@ const retireMissingStations = (existingStations: Station[], newStationsByName: R }; const EPSILON = 0.000000000001; -const stationLocationHasChanged = (oldStation: Station, newStation: CreateStationData) => ( +const stationLocationHasChanged = ( + oldStation: Station, + newStation: CreateStationData +) => // NOTE: We need to compare these numbers with an epsilon value, otherwise we get floating-point precision issues. Math.abs(oldStation.location.coordinates[0] - newStation.lat) < EPSILON || - Math.abs(oldStation.location.coordinates[1] - newStation.lat) < EPSILON -); + Math.abs(oldStation.location.coordinates[1] - newStation.lat) < EPSILON; -const checkThatStationsAreNotTooCloseTogether = (stations: Array) => { - const allStations = stations.map(s => { - if (s.hasOwnProperty('lat')) { +const checkThatStationsAreNotTooCloseTogether = ( + stations: Array +) => { + const allStations = stations.map((s) => { + if (s.hasOwnProperty("lat")) { return { name: (s as CreateStationData).name, - location: [(s as CreateStationData).lat, (s as CreateStationData).lng] as [number, number] + location: [ + (s as CreateStationData).lat, + (s as CreateStationData).lng + ] as [number, number] }; } else { return { name: (s as Station).name, location: (s as Station).location.coordinates - } + }; } }); for (const a of allStations) { for (const b of allStations) { if (a !== b && a.name !== b.name) { - if (latLngApproxDistance(a.location, b.location) < MIN_STATION_SEPARATION_METERS) { + if ( + latLngApproxDistance(a.location, b.location) < + MIN_STATION_SEPARATION_METERS + ) { throw new ClientError("Stations too close together"); } } } } -} +}; -const updateExistingRecordingsForGroupWithMatchingStationsFromDate = async (authUser: User, group: Group, fromDate: Date, stations: Station[]): Promise[]> => { +const updateExistingRecordingsForGroupWithMatchingStationsFromDate = async ( + authUser: User, + group: Group, + fromDate: Date, + stations: Station[] +): Promise[]> => { // Now addedStations are properly resolved with ids: // Now we can look for all recordings in the group back to startDate, and check if any of them // should be assigned to any of our stations. // Get recordings for group starting at date: - const builder = await new AllModels.Recording.queryBuilder().init( - authUser, - { - // Group id, and after date - GroupId: group.id, - createdAt: { - [Op.gte]: fromDate.toISOString() - } - } + const builder = await new AllModels.Recording.queryBuilder().init(authUser, { + // Group id, and after date + GroupId: group.id, + createdAt: { + [Op.gte]: fromDate.toISOString() + } + }); + const recordingsFromStartDate: Recording[] = await AllModels.Recording.findAll( + builder.get() ); - const recordingsFromStartDate: Recording[] = await AllModels.Recording.findAll(builder.get()); const recordingOpPromises = []; // Find matching recordings to apply stations to from `applyToRecordingsFromDate` for (const recording of recordingsFromStartDate) { // NOTE: This await call won't actually block, since we're passing all the stations in. - const matchingStation = await tryToMatchRecordingToStation(recording, stations); + const matchingStation = await tryToMatchRecordingToStation( + recording, + stations + ); if (matchingStation !== null) { recordingOpPromises.push(recording.setStation(matchingStation)); } @@ -107,7 +128,6 @@ const updateExistingRecordingsForGroupWithMatchingStationsFromDate = async (auth return recordingOpPromises; }; - export interface Group extends Sequelize.Model, ModelCommon { id: GroupId; addUser: (userToAdd: User, through: any) => Promise; @@ -252,7 +272,9 @@ export default function (sequelize, DataTypes): GroupStatic { // Enforce name uniqueness to group here: let existingStations: Station[] = await group.getStations(); // Filter out retired stations. - existingStations = existingStations.filter(station => station.retiredAt === null); + existingStations = existingStations.filter( + (station) => station.retiredAt === null + ); const existingStationsByName: Record = {}; const newStationsByName: Record = {}; @@ -263,14 +285,21 @@ export default function (sequelize, DataTypes): GroupStatic { // Make sure existing stations that are not in the current update are retired, and removed from // the list of existing stations that we are comparing with. - const retiredStations = retireMissingStations(existingStations, newStationsByName, authUser.id); + const retiredStations = retireMissingStations( + existingStations, + newStationsByName, + authUser.id + ); for (const station of existingStations) { existingStationsByName[station.name] = station; } // Make sure no two stations are too close to each other: - checkThatStationsAreNotTooCloseTogether([...existingStations, ...stationsToAdd]); + checkThatStationsAreNotTooCloseTogether([ + ...existingStations, + ...stationsToAdd + ]); // Add new stations, or update lat/lng if station with same name but different lat lng. const addedOrUpdatedStations = []; @@ -285,18 +314,21 @@ export default function (sequelize, DataTypes): GroupStatic { }); addedOrUpdatedStations.push(stationToAddOrUpdate); stationOpsPromises.push( - new Promise(async (resolve) => { - await stationToAddOrUpdate.save(); - await group.addStation(stationToAddOrUpdate); - resolve(); - }) + new Promise(async (resolve) => { + await stationToAddOrUpdate.save(); + await group.addStation(stationToAddOrUpdate); + resolve(); + }) ); } else { // Update lat/lng if it has changed but the name is the same stationToAddOrUpdate = existingStationsByName[newStation.name]; if (stationLocationHasChanged(stationToAddOrUpdate, newStation)) { // NOTE - Casting this as "any" because station.location has a special setter function - (stationToAddOrUpdate as any).location = [newStation.lat, newStation.lng]; + (stationToAddOrUpdate as any).location = [ + newStation.lat, + newStation.lng + ]; stationToAddOrUpdate.lastUpdatedById = authUser.id; addedOrUpdatedStations.push(stationToAddOrUpdate); stationOpsPromises.push(stationToAddOrUpdate.save()); @@ -307,10 +339,15 @@ export default function (sequelize, DataTypes): GroupStatic { await Promise.all([...stationOpsPromises, ...retiredStations]); if (applyToRecordingsFromDate) { // After adding stations, we need to apply any station matches to recordings from a start date: - const updatedRecordings = await updateExistingRecordingsForGroupWithMatchingStationsFromDate(authUser, group, applyToRecordingsFromDate, allStations); + const updatedRecordings = await updateExistingRecordingsForGroupWithMatchingStationsFromDate( + authUser, + group, + applyToRecordingsFromDate, + allStations + ); await Promise.all(updatedRecordings); } - return addedOrUpdatedStations.map(({id}) => id); + return addedOrUpdatedStations.map(({ id }) => id); }; /** diff --git a/models/Recording.ts b/models/Recording.ts index 458ff607..78482eae 100644 --- a/models/Recording.ts +++ b/models/Recording.ts @@ -39,11 +39,12 @@ import { GroupId as GroupIdAlias } from "./Group"; import { Track, TrackId } from "./Track"; import jsonwebtoken from "jsonwebtoken"; import { TrackTag } from "./TrackTag"; -import {CreateStationData, Station, StationId} from "./Station"; +import { CreateStationData, Station, StationId } from "./Station"; import { latLngApproxDistance, MAX_DISTANCE_FROM_STATION_FOR_RECORDING, - MIN_STATION_SEPARATION_METERS, tryToMatchRecordingToStation + MIN_STATION_SEPARATION_METERS, + tryToMatchRecordingToStation } from "../api/V1/recordingUtil"; export type RecordingId = number; @@ -754,7 +755,7 @@ from ( Recording.prototype.setStation = async function (station: { id: number }) { this.StationId = station.id; return this.save(); - } + }; Recording.prototype.getFileBaseName = function (): string { return moment(new Date(this.recordingDateTime)) @@ -1413,7 +1414,9 @@ from ( "processingMeta", "GroupId", "DeviceId", - "StationId" + "StationId", + "recordingDateTime", + "location" ]; return Recording; diff --git a/models/Station.ts b/models/Station.ts index 63662c86..cfd415bb 100644 --- a/models/Station.ts +++ b/models/Station.ts @@ -20,7 +20,7 @@ import Sequelize, { BuildOptions, ModelAttributes } from "sequelize"; import { ModelCommon, ModelStaticCommon } from "./index"; import util from "./util/util"; import validation from "./util/validation"; -import {UserId} from "./User"; +import { UserId } from "./User"; export type StationId = number; @@ -35,7 +35,7 @@ export interface Station extends Sequelize.Model, ModelCommon { id: StationId; name: string; location: { - coordinates: [number, number] + coordinates: [number, number]; }; lastUpdatedById: UserId; createdAt: Date; diff --git a/models/User.ts b/models/User.ts index 7ff873d2..fcb7fccf 100644 --- a/models/User.ts +++ b/models/User.ts @@ -293,7 +293,9 @@ export default function ( return groups.map((g) => g.id); }; - User.prototype.isInGroup = async function (groupId: number): Promise { + User.prototype.isInGroup = async function ( + groupId: number + ): Promise { const groupIds = await this.getGroupsIds(); return groupIds.includes(groupId) === true; }; diff --git a/models/index.ts b/models/index.ts index ee727a5f..f9baed1d 100644 --- a/models/index.ts +++ b/models/index.ts @@ -34,7 +34,7 @@ import { GroupStatic } from "./Group"; import { GroupUsersStatic } from "./GroupUsers"; import { DeviceUsersStatic } from "./DeviceUsers"; import { ScheduleStatic } from "./Schedule"; -import {StationStatic} from "./Station"; +import { StationStatic } from "./Station"; const basename = path.basename(module.filename); const dbConfig = config.database;