diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1c90c7f4d1..e4f1844aa1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -57,7 +57,7 @@ jobs: spelling: name: Docs Spelling Checks runs-on: ubuntu-latest - timeout-minutes: 1 + timeout-minutes: 2 steps: - uses: actions/checkout@v4 - name: Check Spelling diff --git a/bin/test_schema b/bin/test_schema index 19f6c3a40f..8d733e8309 100755 --- a/bin/test_schema +++ b/bin/test_schema @@ -15,7 +15,7 @@ upgrade= schemadir=schema testdir=tests/schemas/ outroot=out -rm -rf outroot/tests +rm -rf $outroot/tests while getopts "d:s:fnu" opt; do case $opt in @@ -69,7 +69,7 @@ for subset in $subsets; do reldir=$(realpath --relative-to $schemadir $testdir) reltest=$reldir:$subset fi - cmd="$javarun -a $schemaname $upgrade -f $reltest" + cmd="$javarun -n -a $schemaname $upgrade -f $reltest" echo $cmd (cd $schemadir; $cmd) || true diff --git a/common/src/main/java/com/google/daq/mqtt/util/ExceptionList.java b/common/src/main/java/com/google/daq/mqtt/util/ExceptionList.java new file mode 100644 index 0000000000..c37fccdfa6 --- /dev/null +++ b/common/src/main/java/com/google/daq/mqtt/util/ExceptionList.java @@ -0,0 +1,15 @@ +package com.google.daq.mqtt.util; + +import static java.lang.String.format; + +import java.util.List; + +public class ExceptionList extends RuntimeException { + + private final List list; + + public ExceptionList(List list) { + super(format("List of %d exceptions", list.size())); + this.list = list; + } +} diff --git a/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java b/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java index 8608ef4b2b..621f0a8980 100644 --- a/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java +++ b/common/src/main/java/com/google/udmi/util/CommandLineProcessor.java @@ -1,6 +1,7 @@ package com.google.udmi.util; import static com.google.common.base.Preconditions.checkState; +import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; import static java.lang.String.CASE_INSENSITIVE_ORDER; import static java.util.Objects.nonNull; @@ -119,7 +120,7 @@ public List processArgs(List argList) { } return null; } catch (Exception e) { - showUsage(e.getMessage()); + showUsage(friendlyStackTrace(e)); return null; } } diff --git a/etc/validator.out b/etc/validator.out index bc4884bd22..4d4320d94d 100644 --- a/etc/validator.out +++ b/etc/validator.out @@ -66,7 +66,7 @@ sites/udmi_site_model/out/devices/AHU-1/events_pointset.out "sub_type" : "events", "status" : { "message" : "Multiple validation errors", - "detail" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; While converting to json node: 1 schema violations found", + "detail" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; 1 schema violations found", "category" : "validation.device.multiple", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 @@ -82,8 +82,8 @@ sites/udmi_site_model/out/devices/AHU-1/events_pointset.out "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "events_pointset: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; object instance has properties which are not allowed by the schema: [\"extraField\"]", + "message" : "1 schema violations found", + "detail" : "events_pointset: 1 schema violations found; object instance has properties which are not allowed by the schema: [\"extraField\"]", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 @@ -145,7 +145,7 @@ sites/udmi_site_model/out/devices/AHU-1/state.out "sub_type" : "state", "status" : { "message" : "Multiple validation errors", - "detail" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; While converting to json node: 1 schema violations found", + "detail" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; 1 schema violations found", "category" : "validation.device.multiple", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 @@ -157,8 +157,8 @@ sites/udmi_site_model/out/devices/AHU-1/state.out "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "state_update: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; /system: object instance has properties which are not allowed by the schema: [\"extraField\"]", + "message" : "1 schema violations found", + "detail" : "state_update: 1 schema violations found; /system: object instance has properties which are not allowed by the schema: [\"extraField\"]", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 @@ -411,15 +411,15 @@ sites/udmi_site_model/out/devices/GAT-123/events_discovery.out "sub_folder" : "discovery", "sub_type" : "events", "status" : { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "events_discovery: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; object has missing required properties ([\"scan_family\"])", + "message" : "1 schema violations found", + "detail" : "events_discovery: 1 schema violations found; object has missing required properties ([\"scan_family\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, "errors" : [ { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "events_discovery: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; object has missing required properties ([\"scan_family\"])", + "message" : "1 schema violations found", + "detail" : "events_discovery: 1 schema violations found; object has missing required properties ([\"scan_family\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 diff --git a/tests/schemas/events_discovery/discovery_node.json b/tests/schemas/events_discovery/discovery_node.json new file mode 100644 index 0000000000..45325eeede --- /dev/null +++ b/tests/schemas/events_discovery/discovery_node.json @@ -0,0 +1,245 @@ +{ + "timestamp": "2024-11-21T12:10:46Z", + "version": "1.5.1", + "generation": "2024-11-21T12:09:30Z", + "scan_family": "bacnet", + "scan_addr": "10117", + "event_no": 9, + "families": { + "ipv4": { + "addr": "192.168.1.131" + } + }, + "refs": { + "AI:1": { + "name": "FCU_1180 Room Temp", + "possible_values": [], + "units": "degreesCelsius", + "description": "", + "ancillary": { + "present_value": 20.488115310668945 + } + }, + "AI:2": { + "name": "FCU_1180 Supply Air Temp", + "possible_values": [], + "units": "degreesCelsius", + "description": "", + "ancillary": { + "present_value": 20.14183235168457 + } + }, + "AO:1": { + "name": "FCU_1180 Heating Valve", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 0.0 + } + }, + "AO:2": { + "name": "FCU_1180 Cooling Valve", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 0.0 + } + }, + "AO:3": { + "name": "FCU_1180 Fan Speed", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 70.0 + } + }, + "AV:1": { + "name": "FCU_1180 Setpoint", + "possible_values": [], + "units": "degreesCelsius", + "description": "", + "ancillary": { + "present_value": 20.0 + } + }, + "AV:2": { + "name": "FCU_1180 Min Fan Speed", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 20.0 + } + }, + "AV:3": { + "name": "FCU_1180 Max Fan Speed", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 100.0 + } + }, + "AV:4": { + "name": "FCU_1180 Design Fan Speed", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 70.0 + } + }, + "AV:5": { + "name": "FCU_1180 Design Heating Valve Position", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 75.0 + } + }, + "AV:6": { + "name": "FCU_1180 Design Cooling Valve Position", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 67.0 + } + }, + "AV:7": { + "name": "FCU_1180 Mode_Mimic", + "possible_values": [], + "units": "noUnits", + "description": "", + "ancillary": { + "present_value": 1.0 + } + }, + "AV:9991": { + "name": "FCU_1180 Comfort KPI Score (Live)", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 69.89265441894531 + } + }, + "AV:9992": { + "name": "FCU_1180 Comfort KPI Score (Yesterday)", + "possible_values": [], + "units": "percent", + "description": "", + "ancillary": { + "present_value": 69.99639129638672 + } + }, + "BI:3": { + "name": "FCU_1180 Fan Status", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": "active" + } + }, + "BI:4": { + "name": "FCU_1180 Occupancy", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": "active" + } + }, + "BO:4": { + "name": "FCU_1180 Enable", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": "active" + } + }, + "BV:1": { + "name": "FCU_1180 BMS Remote", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": "active" + } + }, + "BV:2": { + "name": "FCU_1180 FCU Run Status", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": "active" + } + }, + "BV:9991": { + "name": "FCU_1180 Within Bounds", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": "active" + } + }, + "MSV:1": { + "name": "FCU_1180 Mode", + "possible_values": [ + "Auto_", + "On_", + "Off_", + "Flushing_", + "Commissioning_" + ], + "description": "", + "ancillary": { + "present_value": 1 + } + }, + "LP:1": { + "name": "FCU_101 Valve Control Loop", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": 50.0 + } + }, + "LP:2": { + "name": "FCU_101 Fan Control Loop", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": 50.0 + } + }, + "CSV:1": { + "name": "Google Cloud IoT Gateway Configuration", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": "{\n \"hostName\": \"mqtt.bos.goog\",\n \"tcpPort\": 8883,\n \"location\": \"us-central1\",\n \"project\": \"bos-platform-testing\",\n \"registry\": \"UK-LON-GLAB\",\n \"device\": \"DDC-17\"\n}" + } + }, + "CSV:1000001": { + "name": "BLE PIN", + "possible_values": [], + "description": "", + "ancillary": { + "present_value": "800700" + } + } + }, + "system": { + "serial_no": "141557/0009", + "ancillary": { + "name": "DDC-17" + }, + "hardware": { + "make": "Delta Controls Inc.", + "model": "Red5-PLUS-1180" + } + } +} diff --git a/tests/schemas/events_discovery/discovery_node.out b/tests/schemas/events_discovery/discovery_node.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/schemas/events_discovery/node.out b/tests/schemas/events_discovery/node.out index ba111b3d96..98fe9b373a 100644 --- a/tests/schemas/events_discovery/node.out +++ b/tests/schemas/events_discovery/node.out @@ -1,2 +1,2 @@ 1 schema violations found - /system: object instance has properties which are not allowed by the schema: ["software"] \ No newline at end of file + /system: object instance has properties which are not allowed by the schema: ["software"] diff --git a/tests/traces/simple/expected/devices/AHU-22/events_pointset.out b/tests/traces/simple/expected/devices/AHU-22/events_pointset.out index 88f68e21a2..1806aafde7 100644 --- a/tests/traces/simple/expected/devices/AHU-22/events_pointset.out +++ b/tests/traces/simple/expected/devices/AHU-22/events_pointset.out @@ -4,8 +4,8 @@ "sub_folder" : "pointset", "sub_type" : "events", "status" : { - "message" : "While converting to json node: 2 schema violations found", - "detail" : "events_pointset: While converting to json node: 2 schema violations found @Validator.validateMessage(REDACTED); 2 schema violations found; object instance has properties which are not allowed by the schema: [\"yesvalue\"]; object has missing required properties ([\"version\"])", + "message" : "2 schema violations found", + "detail" : "events_pointset: 2 schema violations found; object instance has properties which are not allowed by the schema: [\"yesvalue\"]; object has missing required properties ([\"version\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 @@ -15,8 +15,8 @@ "extra" : [ ] }, "errors" : [ { - "message" : "While converting to json node: 2 schema violations found", - "detail" : "events_pointset: While converting to json node: 2 schema violations found @Validator.validateMessage(REDACTED); 2 schema violations found; object instance has properties which are not allowed by the schema: [\"yesvalue\"]; object has missing required properties ([\"version\"])", + "message" : "2 schema violations found", + "detail" : "events_pointset: 2 schema violations found; object instance has properties which are not allowed by the schema: [\"yesvalue\"]; object has missing required properties ([\"version\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 diff --git a/tests/traces/simple/expected/devices/SNS-4/events_pointset.out b/tests/traces/simple/expected/devices/SNS-4/events_pointset.out index ae6aa7735e..bc6ade3ecc 100644 --- a/tests/traces/simple/expected/devices/SNS-4/events_pointset.out +++ b/tests/traces/simple/expected/devices/SNS-4/events_pointset.out @@ -5,7 +5,7 @@ "sub_type" : "events", "status" : { "message" : "Multiple validation errors", - "detail" : "While converting to json node: 1 schema violations found; Device has extra points: rocket_count, triangulating_axis_y", + "detail" : "1 schema violations found; Device has extra points: rocket_count, triangulating_axis_y", "category" : "validation.device.multiple", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 @@ -15,8 +15,8 @@ "extra" : [ "rocket_count", "triangulating_axis_y" ] }, "errors" : [ { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "events_pointset: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; object instance has properties which are not allowed by the schema: [\"monkey\"]", + "message" : "1 schema violations found", + "detail" : "events_pointset: 1 schema violations found; object instance has properties which are not allowed by the schema: [\"monkey\"]", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 diff --git a/tests/traces/simple/expected/devices/SNS-4/events_system.out b/tests/traces/simple/expected/devices/SNS-4/events_system.out index 6c5c8c1f7f..b007b15d45 100644 --- a/tests/traces/simple/expected/devices/SNS-4/events_system.out +++ b/tests/traces/simple/expected/devices/SNS-4/events_system.out @@ -4,15 +4,15 @@ "sub_folder" : "system", "sub_type" : "events", "status" : { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "events_system: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; /logentries/0/category: instance entry category not recognized", + "message" : "1 schema violations found", + "detail" : "events_system: 1 schema violations found; /logentries/0/category: instance entry category not recognized", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 }, "errors" : [ { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "events_system: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; /logentries/0/category: instance entry category not recognized", + "message" : "1 schema violations found", + "detail" : "events_system: 1 schema violations found; /logentries/0/category: instance entry category not recognized", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 diff --git a/tests/traces/simple/expected/validation_report.json b/tests/traces/simple/expected/validation_report.json index 73ab4b9b83..dfa095c54e 100644 --- a/tests/traces/simple/expected/validation_report.json +++ b/tests/traces/simple/expected/validation_report.json @@ -25,8 +25,8 @@ "AHU-22" : { "last_seen" : "1999-10-20T01:02:03Z", "status" : { - "message" : "While converting to json node: 2 schema violations found", - "detail" : "events_pointset: While converting to json node: 2 schema violations found @Validator.validateMessage(REDACTED); 2 schema violations found; object instance has properties which are not allowed by the schema: [\"yesvalue\"]; object has missing required properties ([\"version\"])", + "message" : "2 schema violations found", + "detail" : "events_pointset: 2 schema violations found; object instance has properties which are not allowed by the schema: [\"yesvalue\"]; object has missing required properties ([\"version\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 @@ -46,7 +46,7 @@ "last_seen" : "1999-10-20T01:02:03Z", "status" : { "message" : "Multiple validation errors", - "detail" : "While converting to json node: 1 schema violations found; Device has extra points: rocket_count, triangulating_axis_y; Unexpected character ('{' (code 123)): was expecting double-quote to start field name\n at [Source: (File), line: 1, column: 3]; While converting to json node: 1 schema violations found", + "detail" : "1 schema violations found; Device has extra points: rocket_count, triangulating_axis_y; Unexpected character ('{' (code 123)): was expecting double-quote to start field name\n at [Source: (File), line: 1, column: 3]; 1 schema violations found", "category" : "validation.device.multiple", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 diff --git a/tests/traces/upgrade/expected/devices/AHU-1/state.out b/tests/traces/upgrade/expected/devices/AHU-1/state.out index 1bd549ce5d..82ae93dc82 100644 --- a/tests/traces/upgrade/expected/devices/AHU-1/state.out +++ b/tests/traces/upgrade/expected/devices/AHU-1/state.out @@ -5,14 +5,14 @@ "sub_type" : "state", "status" : { "message" : "Multiple validation errors", - "detail" : "While converting to json node: 1 schema violations found; missing pointset subblock", + "detail" : "1 schema violations found; missing pointset subblock", "category" : "validation.device.multiple", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 }, "errors" : [ { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "state_update: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", + "message" : "1 schema violations found", + "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 diff --git a/tests/traces/upgrade/expected/devices/AHU-1/state_system.out b/tests/traces/upgrade/expected/devices/AHU-1/state_system.out index 1ad02b544f..4a74512835 100644 --- a/tests/traces/upgrade/expected/devices/AHU-1/state_system.out +++ b/tests/traces/upgrade/expected/devices/AHU-1/state_system.out @@ -4,15 +4,15 @@ "sub_folder" : "system", "sub_type" : "state", "status" : { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "state_system: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; object has missing required properties ([\"serial_no\"])", + "message" : "1 schema violations found", + "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 }, "errors" : [ { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "state_system: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; object has missing required properties ([\"serial_no\"])", + "message" : "1 schema violations found", + "detail" : "state_system: 1 schema violations found; object has missing required properties ([\"serial_no\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 diff --git a/tests/traces/upgrade/expected/devices/AHU-22/state.out b/tests/traces/upgrade/expected/devices/AHU-22/state.out index 1bd549ce5d..82ae93dc82 100644 --- a/tests/traces/upgrade/expected/devices/AHU-22/state.out +++ b/tests/traces/upgrade/expected/devices/AHU-22/state.out @@ -5,14 +5,14 @@ "sub_type" : "state", "status" : { "message" : "Multiple validation errors", - "detail" : "While converting to json node: 1 schema violations found; missing pointset subblock", + "detail" : "1 schema violations found; missing pointset subblock", "category" : "validation.device.multiple", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 }, "errors" : [ { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "state_update: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", + "message" : "1 schema violations found", + "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 diff --git a/tests/traces/upgrade/expected/devices/NON-1/state.out b/tests/traces/upgrade/expected/devices/NON-1/state.out index caa8884df0..4c65baed52 100644 --- a/tests/traces/upgrade/expected/devices/NON-1/state.out +++ b/tests/traces/upgrade/expected/devices/NON-1/state.out @@ -4,15 +4,15 @@ "sub_folder" : "update", "sub_type" : "state", "status" : { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "state_update: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", + "message" : "1 schema violations found", + "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 }, "errors" : [ { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "state_update: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", + "message" : "1 schema violations found", + "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", "category" : "validation.device.schema", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 diff --git a/tests/traces/upgrade/expected/validation_report.json b/tests/traces/upgrade/expected/validation_report.json index c6905a1986..7c4656ff83 100644 --- a/tests/traces/upgrade/expected/validation_report.json +++ b/tests/traces/upgrade/expected/validation_report.json @@ -17,7 +17,7 @@ "last_seen" : "1999-10-20T01:02:03Z", "status" : { "message" : "Multiple validation errors", - "detail" : "While converting to json node: 1 schema violations found; missing pointset subblock; While converting to json node: 1 schema violations found", + "detail" : "1 schema violations found; missing pointset subblock; 1 schema violations found", "category" : "validation.device.multiple", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 @@ -27,7 +27,7 @@ "last_seen" : "1999-10-20T01:02:03Z", "status" : { "message" : "Multiple validation errors", - "detail" : "While converting to json node: 1 schema violations found; missing pointset subblock", + "detail" : "1 schema violations found; missing pointset subblock", "category" : "validation.device.multiple", "timestamp" : "1999-10-20T01:02:03Z", "level" : 500 @@ -36,8 +36,8 @@ "NON-1" : { "last_seen" : "1999-10-20T01:02:03Z", "status" : { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "state_update: While converting to json node: 1 schema violations found @Validator.validateMessage(REDACTED); 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", + "message" : "1 schema violations found", + "detail" : "state_update: 1 schema violations found; /system: object has missing required properties ([\"serial_no\"])", "category" : "validation.device.extra", "timestamp" : "1999-10-20T01:02:03Z", "level" : 400 diff --git a/validator/.idea/runConfigurations/Schema_Tester.xml b/validator/.idea/runConfigurations/Schema_Tester.xml new file mode 100644 index 0000000000..bfff9219ec --- /dev/null +++ b/validator/.idea/runConfigurations/Schema_Tester.xml @@ -0,0 +1,11 @@ + + + + \ No newline at end of file diff --git a/validator/.idea/runConfigurations/Test_Trace_Simple.xml b/validator/.idea/runConfigurations/Test_Trace_Simple.xml new file mode 100644 index 0000000000..fc7dadc5dc --- /dev/null +++ b/validator/.idea/runConfigurations/Test_Trace_Simple.xml @@ -0,0 +1,11 @@ + + + + \ No newline at end of file diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java b/validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java index fefa01b96f..ba67641497 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java @@ -2,10 +2,12 @@ import static com.google.udmi.util.Common.SUBFOLDER_PROPERTY_KEY; import static com.google.udmi.util.Common.SUBTYPE_PROPERTY_KEY; +import static java.lang.String.format; import static java.util.Optional.ofNullable; import static org.junit.Assert.assertTrue; import com.google.common.base.Joiner; +import com.google.daq.mqtt.util.ExceptionList; import com.google.daq.mqtt.util.ValidationException; import com.google.udmi.util.Common; import java.time.Instant; @@ -39,6 +41,7 @@ public class ReportingDevice { private static Date mockNow; private final String deviceId; private final List entries = new ArrayList<>(); + private final Map entryExceptions = new HashMap<>(); private final List messageEntries = new ArrayList<>(); private final Map messageMarks = new HashMap<>(); private Date lastSeen = new Date(0); // Always defined, just start a long time ago! @@ -228,7 +231,7 @@ public void updateLastSeen(Date timestamp) { private Exception pointValidationError(String description, Set points) { return new ValidationException( - String.format("Device has %s: %s", description, Joiner.on(", ").join(points))); + format("Device has %s: %s", description, Joiner.on(", ").join(points))); } private void addEntry(Entry entry) { @@ -250,16 +253,18 @@ void addError(Exception error, Map attributes, String category) String subFolder = attributes.get(SUBFOLDER_PROPERTY_KEY); String subType = attributes.get(SUBTYPE_PROPERTY_KEY); addError(error, category, - String.format("%s: %s", typeFolderPairKey(subType, subFolder), + format("%s: %s", typeFolderPairKey(subType, subFolder), Common.getExceptionDetail(error, this.getClass(), ReportingDevice::validationMessage))); } void addError(Exception error, String category, String detail) { - addEntry(makeEntry(error, category, detail)); + Entry entry = makeEntry(error, category, detail); + entryExceptions.put(entry, error); + addEntry(entry); } public static String typeFolderPairKey(String subType, String subFolder) { - return String.format("%s_%s", ofNullable(subType).orElse(SubType.EVENTS.value()), subFolder); + return format("%s_%s", ofNullable(subType).orElse(SubType.EVENTS.value()), subFolder); } /** @@ -344,6 +349,20 @@ public List getMessageEntries() { return messageEntries; } + /** + * Check for errors and throw a fit if necessary. + */ + public void throwIfFailure() { + if (!hasErrors()) { + return; + } + if (messageEntries.size() == 1) { + throw (RuntimeException) entryExceptions.get(messageEntries.get(0)); + } + List exceptions = entries.stream().map(entry -> entryExceptions.get(entry)).toList(); + throw new ExceptionList(exceptions); + } + /** * Encapsulation of metadata differences. */ diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index 8e0f8f71e1..0bd335f7ae 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -43,7 +43,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration; import com.github.fge.jsonschema.core.load.download.URIDownloader; import com.github.fge.jsonschema.core.report.LogLevel; @@ -186,8 +185,10 @@ public class Validator { private static final int TIMESTAMP_JITTER_SEC = 60; private static final String UDMI_CONFIG_JSON_FILE = "udmi_config.json"; private static final String TOOL_NAME = "validator"; - public static final String VALIDATOR_TOOL_NAME = "validator"; - public static final String REGISTRY_DEVICE_DEFAULT = "_regsitry"; + private static final String FAUX_DEVICE_ID = "TEST-97"; + private static final String VALIDATOR_TOOL_NAME = "validator"; + private static final String REGISTRY_DEVICE_DEFAULT = "_regsitry"; + private static final String SCHEMA_NAME_KEY = "ignore_envelope"; private long reportingDelaySec = DEFAULT_INTERVAL_SEC; private final CommandLineProcessor commandLineProcessor = new CommandLineProcessor(this); private final Map reportingDevices = new TreeMap<>(); @@ -558,8 +559,8 @@ private Map getSchemaMap() { throw new RuntimeException("Missing schema for attribute validation: " + ENVELOPE_SCHEMA_ID); } - // Rename the metadata schema to model, which is how it's handled programmatically. - schemaMap.put("model", schemaMap.remove("metadata")); + // Copy the metadata schema to model, since sometimes it's referenced that way. + schemaMap.put("model", schemaMap.get("metadata")); return schemaMap; } @@ -693,12 +694,9 @@ private void validateMessage(Object msgObject, Map attributes) { } } - private void validateMessage(JsonSchema schema, Object message) { - try { - validateJsonNode(schema, OBJECT_MAPPER.valueToTree(message)); - } catch (Exception e) { - throw new RuntimeException("While converting to json node: " + e.getMessage(), e); - } + private void validateMessage(JsonSchema schema, Object message) throws Exception { + ProcessingReport report = schema.validate(OBJECT_MAPPER.valueToTree(message), true); + ifTrueThen(!report.isSuccess(), () -> ifNotNullThrow(fromProcessingReport(report))); } private Instant getInstant(Object msgObject, Map attributes) { @@ -810,14 +808,18 @@ private boolean processExceptions(Map attributes, String deviceI */ public void validateDeviceMessage(ReportingDevice device, Map message, Map attributes) { - String schemaName = messageSchema(attributes); + String schemaName = ofNullable(attributes.get(SCHEMA_NAME_KEY)).orElseGet( + () -> messageSchema(attributes)); upgradeMessage(schemaName, message); - try { - validateMessage(schemaMap.get(ENVELOPE_SCHEMA_ID), (Object) attributes); - } catch (Exception e) { - outputLogger.error("Error validating attributes: " + friendlyStackTrace(e)); - device.addError(e, attributes, Category.VALIDATION_DEVICE_RECEIVE); + // Assume the attributes know what they're doing when the schema name is provided explicitly. + if (!attributes.containsKey(SCHEMA_NAME_KEY)) { + try { + validateMessage(schemaMap.get(ENVELOPE_SCHEMA_ID), (Object) attributes); + } catch (Exception e) { + outputLogger.error("Error validating attributes: " + friendlyStackTrace(e)); + device.addError(e, attributes, Category.VALIDATION_DEVICE_RECEIVE); + } } if (schemaMap.containsKey(schemaName)) { @@ -1193,7 +1195,6 @@ private void validateFilesOutput(String targetSpec) { } catch (ExceptionMap processingException) { ErrorTree errorTree = ExceptionMap.format(processingException); errorTree.write(System.err); - throw processingException; } } @@ -1253,7 +1254,9 @@ private void validateFile( outputStream.close(); FileUtils.copyFile(inputFile, outputFile); } - validateJsonNode(schema, jsonNode); + ReportingDevice reportingDevice = new ReportingDevice(FAUX_DEVICE_ID); + validateDeviceMessage(reportingDevice, message, makeFileAttributes(schemaName)); + reportingDevice.throwIfFailure(); writeExceptionOutput(targetOut, null); } catch (Exception e) { writeExceptionOutput(targetOut, e); @@ -1261,6 +1264,12 @@ private void validateFile( } } + private Map makeFileAttributes(String schemaName) { + HashMap attributes = new HashMap<>(); + attributes.put(SCHEMA_NAME_KEY, schemaName); + return attributes; + } + private void copyFileHeader(File inputFile, OutputStream outputFile) { try (Scanner scanner = new Scanner(inputFile)) { while (scanner.hasNextLine()) { @@ -1304,11 +1313,6 @@ private void upgradeMessage(String schemaName, Map message) { message.putAll(objectMap); } - private void validateJsonNode(JsonSchema schema, JsonNode jsonNode) throws ProcessingException { - ProcessingReport report = schema.validate(jsonNode, true); - ifTrueThen(!report.isSuccess(), () -> ifNotNullThrow(fromProcessingReport(report))); - } - private File getFullPath(String prefix, File targetFile) { return prefix == null ? targetFile : new File(new File(prefix), targetFile.getPath()); }