Bump up version in files to 1.5.1 (#921)

bin/upgrade_version

@@ -100,7 +100,7 @@ fi
 
 # Check files in upversion list still exist
 while read action file comment; do
-    if [[ $action =~ "^[?!# ]" ]] && ! [[ -f $file ]]; then
+    if [[ $action =~ ^[yn]$ ]] && [[ ! -f $file ]]; then
         err_stale_file=1
         echo $file
     fi
@@ -110,7 +110,6 @@ if [[ -n $err_stale_file ]]; then
     echo ERROR .. above files $UPVERSION_LIST no longer exist
 fi
 
-
 # Check files have exactly 1 matched version fields to avoid unexpected 
 # updates. Uses indents to try and match the top level field
 while read action file comment; do

common/src/main/java/com/google/udmi/util/MessageUpgrader.java

@@ -16,6 +16,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import udmi.schema.SystemModel;
 
 /**
  * Container class for upgrading UDMI messages from older versions.
@@ -147,17 +148,25 @@ private Object upgradeRaw(boolean forceUpgrade) {
       minor = 5;
     }
 
+    if (minor == 5 && patch == 0) {
+      JsonNode before = message.deepCopy();
+      upgradeTo_1_5_1();
+      upgraded |= !before.equals(message);
+      patch = 1;
+      minor = 5;
+    }
+
     if (upgraded && message.get(VERSION_KEY) != null) {
       message.put(UPGRADED_FROM, originalVersion);
       message.put(VERSION_KEY, String.format(TARGET_FORMAT, major, minor, patch));
-    } else {
-      // Even if the message was not modified, it is now conformant to the current version
-      // of UDMI, so update the version property if it exists
-      if (message.has(VERSION_KEY)){
-        message.put(VERSION_KEY, SchemaVersion.CURRENT.key());
-      }
-    }
+    } 
 
+    // Even if the message was not modified, it is now conformant to the current version
+    // of UDMI, so update the version property if it exists
+    if (message.has(VERSION_KEY)){
+      message.put(VERSION_KEY, SchemaVersion.CURRENT.key());
+    }
+
     return message;
   }
 
@@ -206,6 +215,25 @@ private void upgradeTo_1_4_1() {
     }
   }
 
+  private void upgradeTo_1_5_1() {
+    if (METADATA_SCHEMA.equals(schemaName)) {
+      upgradeTo_1_5_1_metadata();
+    }
+  }
+
+  private void upgradeTo_1_5_1_metadata() {
+    JsonNode tags = message.remove("tags");
+    if (tags == null) {
+      return;
+    }
+
+    if (!message.has("system")) {
+      message.put("system", new ObjectNode(NODE_FACTORY));
+    }
+
+    ObjectNode system = (ObjectNode) message.get("system");
+    system.put("tags", tags);
+  }
    private void upgradeTo_1_5_0() {
     if (STATE_SCHEMA.equals(schemaName)) {
       upgradeTo_1_5_0_state();

common/src/main/java/com/google/udmi/util/SchemaVersion.java

@@ -10,6 +10,7 @@
  * Formalized enums of the UDMI schema version.
  */
 public enum SchemaVersion {
+  VERSION_1_5_1("1.5.1", 15010),
   VERSION_1_5_0("1.5.0", 15000),
   VERSION_1_4_2("1.4.2", 14020),
   VERSION_1_4_1("1.4.1", 14010),

docs/guides/development.md

@@ -42,13 +42,20 @@ which are preceded by a `y` in `upversion.txt` will have their version upgraded.
 
 `bin/upgrade_version` does not update any generated files (e.g. for CI testing).
 
+An entry for the new version must be added manually in `util/SchemaVersion.java`
+
 The below files need to be updated. Do not blindly copy! Inspect all diffs and confirm they are expected
 * After `bin/test_trace simple`, contents of `sites/udmi_site_model/sites/out`
   into `tests/traces/simple/expected` 
-* After `bin/test_validator`, `/tmp/validator.out` into `/etc/validator.out` (reset any changes to sites/udmi_site_model before running but run `bin/registrar`) 
+* After `bin/test_validator`, `/tmp/validator.out` into `/etc/validator.out` (reset any changes to sites/udmi_site_model first and and run `bin/registrar`) 
 * After `bin/test_registrar && bin/test_sites`, the `out` directory for each device in `tests/downgrade.site/devices/` into the `expected` subdirectory
   (note these files are ignored by git, but must still be committed)
 
+**Useful commands**
+
+* `rm -rf tests/sites/*/**/*/out && rm -rf tests/sites/*/out` to remove out files from `bin/test_registrar`
+
+
 ## Configuring Cloud CI Tests
 
 To enable the CI tests, there first needs to be a dedicated GCP Project with an IoT Core

docs/specs/sequences/endpoint_reconfiguration.md

@@ -78,7 +78,7 @@ Config message to initiate Reconfiguration (sequence #1 in diagrams above)
 <!--example:config/endpoint_reconfiguration.json-->
 ```json
 {
-  "version": "1.5.0",
+  "version": "1.5.1",
   "blobset": {
     "blobs": {
       "_iot_endpoint_config": {
@@ -108,7 +108,7 @@ a successful reconfiguration
 <!--example:state/endpoint_reconfiguration.json-->
 ```json
 {
-  "version": "1.5.0",
+  "version": "1.5.1",
   "timestamp": "2022-07-13T12:00:10.000Z",
   "system": {
     "hardware": {
@@ -138,7 +138,7 @@ This is an example of the state message sent to the original endpoint after a fa
 <!--example:state/endpoint_reconfiguration_failed.json-->
 ```json
 {
-  "version": "1.5.0",
+  "version": "1.5.1",
   "timestamp": "2022-07-13T12:00:11.000Z",
   "system": {
     "hardware": {

etc/category.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Category",
   "$comment": "Auto-generated category mappings from bin/gencode_categories.",
   "type": "string",

etc/upversion.txt

@@ -86,14 +86,12 @@ y tests/schemas/config/smartprimus.json
 y tests/schemas/config/restart.json
 y tests/schemas/config/implicit.json
 y tests/schemas/state_mapping/mapping.json
-y tests/schemas/command_discovery/provision.json
 y tests/schemas/events_mapping/mapping.json
 y tests/schemas/events_mapping/prediction.json
 y tests/schemas/metadata/toomany.json
 y tests/schemas/metadata/proxy.json
 y tests/schemas/metadata/example.json
 y tests/schemas/metadata/gateway.json
-y tests/schemas/command_mapping/mapping.json
 y tests/schemas/metadata/errors.json
 y tests/schemas/config/blobset_final.json
 n tests/schemas/configuration_execution/cloud_iot_config.json
@@ -103,7 +101,6 @@ n tests/schemas/configuration_endpoint/multiple.json
 n tests/schemas/events_discovery/features.json
 y tests/schemas/events_system/categories.json
 y tests/schemas/metadata/nosys.json
-y tests/schemas/state/gateway_test.json
 y tests/schemas/state_validation/sequencer.json
 n tests/schemas/configuration_pubber/mosquitto.json
 y tests/schemas/metadata/network.json
@@ -156,14 +153,7 @@ n tests/sites/downgrade/cloud_iot_config.json
 n tests/sites/downgrade/devices/DWN-1/expected/generated_config.json
 n tests/sites/downgrade/devices/DWN-1/expected/metadata_norm.json
 n tests/sites/downgrade/devices/DWN-1/metadata.json
-n tests/sites/downgrade/devices/DWN-1/out/errors.json
-n tests/sites/downgrade/devices/DWN-1/out/generated_config.json
-n tests/sites/downgrade/devices/DWN-1/out/metadata_norm.json
 n tests/sites/downgrade/devices/DWN-2/metadata.json
-n tests/sites/downgrade/devices/DWN-2/out/errors.json
-n tests/sites/downgrade/devices/DWN-2/out/generated_config.json
-n tests/sites/downgrade/devices/DWN-2/out/metadata_norm.json
-n tests/sites/downgrade/out/registration_summary.json
 n tests/sites/downgrade/registration_summary.json
 n tests/sites/downgrade/site_defaults.json
 
@@ -172,25 +162,58 @@ n tests/sites/missing/cloud_iot_config.json
 n tests/sites/missing/devices/AHU-1/expected/generated_config.json
 n tests/sites/missing/devices/AHU-1/expected/metadata_norm.json
 n tests/sites/missing/devices/AHU-1/metadata.json
-n tests/sites/missing/devices/AHU-1/out/errors.json
-n tests/sites/missing/devices/AHU-1/out/generated_config.json
-n tests/sites/missing/devices/AHU-1/out/metadata_norm.json
-n tests/sites/missing/devices/AHU-22/expected/generated_config.json
 n tests/sites/missing/devices/AHU-22/expected/metadata_norm.json
 n tests/sites/missing/devices/AHU-22/metadata.json
-n tests/sites/missing/devices/AHU-22/out/generated_config.json
-n tests/sites/missing/devices/AHU-22/out/metadata_norm.json
 n tests/sites/missing/devices/GAT-123/expected/generated_config.json
 n tests/sites/missing/devices/GAT-123/expected/metadata_norm.json
 n tests/sites/missing/devices/GAT-123/metadata.json
-n tests/sites/missing/devices/GAT-123/out/errors.json
-n tests/sites/missing/devices/GAT-123/out/generated_config.json
-n tests/sites/missing/devices/GAT-123/out/metadata_norm.json
 n tests/sites/missing/devices/SNS-4/expected/generated_config.json
 n tests/sites/missing/devices/SNS-4/expected/metadata_norm.json
 n tests/sites/missing/devices/SNS-4/metadata.json
-n tests/sites/missing/devices/SNS-4/out/generated_config.json
-n tests/sites/missing/devices/SNS-4/out/metadata_norm.json
-n tests/sites/missing/out/registration_summary.json
+n tests/sites/missing/site_defaults.json
 n tests/sites/missing/registration_summary.json
-n tests/sites/missing/site_defaultsjson
+
+n tests/sites/configs/cloud_iot_config.json
+y tests/sites/configs/devices/AHU-1/expected/metadata_norm.json
+y tests/sites/configs/devices/AHU-1/metadata.json
+y tests/sites/configs/devices/AHU-22/expected/generated_config.json
+y tests/sites/configs/devices/AHU-22/expected/metadata_norm.json
+y tests/sites/configs/devices/AHU-22/metadata.json
+n tests/sites/configs/devices/GAT-123/config/config.json # static config 
+n tests/sites/configs/devices/GAT-123/expected/generated_config.json # static config
+y tests/sites/configs/devices/GAT-123/expected/metadata_norm.json
+y tests/sites/configs/devices/GAT-123/metadata.json
+n tests/sites/configs/devices/GAT-456/expected/generated_config.json # static config
+y tests/sites/configs/devices/GAT-456/expected/metadata_norm.json
+y tests/sites/configs/devices/GAT-456/metadata.json
+n tests/sites/configs/registration_summary.json
+n tests/sites/discovery/cloud_iot_config.json
+y tests/sites/discovery/devices/GAT-123/expected/generated_config.json
+y tests/sites/discovery/devices/GAT-123/expected/metadata_norm.json
+y tests/sites/discovery/devices/GAT-123/metadata.json
+n tests/sites/discovery/registration_summary.json
+n tests/sites/downgrade/devices/AHU-102/expected/generated_config.json
+n tests/sites/downgrade/devices/AHU-102/expected/metadata_norm.json
+n tests/sites/downgrade/devices/AHU-102/metadata.json
+n tests/sites/downgrade/devices/CGW-1/expected/generated_config.json # downgraded config
+n tests/sites/downgrade/devices/CGW-1/expected/metadata_norm.json
+n tests/sites/downgrade/devices/CGW-1/metadata.json
+n tests/sites/downgrade/devices/CGW-12/expected/generated_config.json # downgraded config
+n tests/sites/downgrade/devices/CGW-12/expected/metadata_norm.json
+n tests/sites/downgrade/devices/CGW-12/metadata.json
+n tests/sites/downgrade/devices/DDC-54/expected/generated_config.json # downgraded config
+n tests/sites/downgrade/devices/DDC-54/expected/metadata_norm.json
+n tests/sites/downgrade/devices/DDC-54/metadata.json
+n tests/sites/downgrade/devices/DWN-3/expected/generated_config.json # downgraded config
+n tests/sites/downgrade/devices/DWN-3/expected/metadata_norm.json
+n tests/sites/downgrade/devices/DWN-3/metadata.json
+n tests/sites/downgrade/devices/DWN-4/metadata.json
+n tests/sites/downgrade/devices/SF-28/expected/generated_config.json # downgraded config
+n tests/sites/downgrade/devices/SF-28/expected/metadata_norm.json
+n tests/sites/downgrade/devices/SF-28/metadata.json
+
+# These files do not have versions in them
+n tests/sites/mapping/cloud_iot_config.json
+n tests/sites/mapping/devices/GAT-123/expected/metadata_norm.json
+n tests/sites/mapping/devices/GAT-123/metadata.json
+n tests/sites/mapping/registration_summary.json

schema/access_iot.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "type": "object",
   "title": "Iot Access",
   "additionalProperties": false,

schema/ancillary_properties.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Ancillary Properties",
   "description": "Arbitrary blob of json associated with this point",
   "existingJavaType": "java.util.Map<String, Object>",

schema/building_translation.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Building Translation",
   "description": "[Discovery result](../docs/specs/discovery.md) with implicit enumeration",
   "type": "object",

schema/category.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Category",
   "$comment": "Auto-generated category mappings from bin/gencode_categories.",
   "type": "string",

schema/commands_discovery.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Discovery Command",
   "description": "[Discovery command](../docs/specs/discovery.md) for provisioning",
   "type": "object",

schema/commands_mapping.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Mapping Command",
   "description": "[Mapping command](../docs/specs/mapping.md) for provisioning",
   "type": "object",

schema/common.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Common",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",

schema/config.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Config",
   "description": "The config block controls a device's intended behavior. [Config Documentation](../docs/messages/config.md)",
   "$section": "Messages",

schema/config_blobset.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Blobset Config",
   "type": "object",
   "$schema": "http://json-schema.org/draft-07/schema#",

schema/config_blobset_blob.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Blob Blobset Config",
   "type": "object",
   "$schema": "http://json-schema.org/draft-07/schema#",

schema/config_discovery.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Discovery Config",
   "description": "Configuration for [discovery](../docs/specs/discovery.md)",
   "type": "object",

schema/config_discovery_family.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Family Discovery Config",
   "description": "Configuration for [discovery](../docs/specs/discovery.md)",
   "type": "object",

schema/config_gateway.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Gateway Config",
   "description": "Configuration for gateways. Only required for devices which are acting as [gateways](../docs/specs/gateway.md)",
   "type": "object",

schema/config_localnet.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Localnet Config",
   "description": "Currently unused: request local network configuration",
   "type": "object",

schema/config_localnet_family.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Family Localnet Config",
   "type": "object",
   "$schema": "http://json-schema.org/draft-07/schema#",

schema/config_mapping.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Mapping Config",
   "description": "Configuration for [mapping](../docs/specs/mapping.md)",
   "type": "object",

schema/config_mapping_device.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Device Mapping Config",
   "description": "Configuration for [mapping](../docs/specs/mapping.md)",
   "type": "object",

schema/config_pointset.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Pointset Config",
   "description": "[Pointset Config Documentation](../docs/messages/pointset.md#config)",
   "type": "object",

schema/config_pointset_point.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "Point Pointset Config",
   "type": "object",
   "$schema": "http://json-schema.org/draft-07/schema#",

schema/config_system.json

@@ -1,5 +1,5 @@
 {
-  "$udmi_version": "1.5.0",
+  "$udmi_version": "1.5.1",
   "title": "System Config",
   "description": "[System Config Documentation](../docs/messages/system.md#config)",
   "type": "object",