Add generation timestamp to registrar metadata

.gencode_hash.txt

@@ -15,7 +15,7 @@ eafcc3c48189f605f114cde051fea9d13fc5f1a3e395d64fb0a91cb53d4c9aeb  gencode/docs/e
 81043ba732caab75704d38e6977b32b1c2bcdad89d94ad2a259d0e70ca4aa40e  gencode/docs/events_system.html
 46fd37b577c655965346d1e3ececf559c4384791214900eeeea1ad8fc72a02b8  gencode/docs/events_udmi.html
 baf59590b1a71a49e59d8f499ae5112a1b7a555294dbd72bc821203aecceefa3  gencode/docs/events_validation.html
-d41d92237c191cc32c4b734201f221a07ac44db528f99524a91cbe23571bfdd5  gencode/docs/metadata.html
+bf60f5a343a57de94d74326dfb9204c73918cb1afcd394a12966565fcd1c042a  gencode/docs/metadata.html
 9277115c2937b52954c565fd2701cdd03e74ec2cb16b6567490770b8df2753f3  gencode/docs/monitoring.html
 6f332da9cc64230df3551f9748a4eaa4060146e767cae967fc0e73b1f348be5b  gencode/docs/persistent_device.html
 e11595fd11477947a27461f8ef4fb6facb5f60e2abd6212193f7581ab123ff84  gencode/docs/properties.html
@@ -106,7 +106,7 @@ fc2e671ab0c4b15ff0ea3f7db417d969bd5af8e86c00b8acef4f2e9d34fea1d2  gencode/java/u
 ce36448574f7e3ae85523b6cdc580c43668667ab96f9dcfdfb3bf3075bebb806  gencode/java/udmi/schema/MappingEvents.java
 41a49d92e0b66a84c7ae4cecd665d9c088350cac83b0a14483619f5c467ec638  gencode/java/udmi/schema/MappingState.java
 faab64907b7d5a7ee6aeb8f6f2926274b8bf7a732e27ca28b1bffa64ca44c93d  gencode/java/udmi/schema/MessageTemplateData.java
-8dde7cc44b13ef463eea2bafc7cbbe17cb60e6f305cedb0a166a21608f6f2796  gencode/java/udmi/schema/Metadata.java
+65f839bb27a28cc44e96573dbe6793822408fdfad86f1b7b4cef62ba4d5eba9d  gencode/java/udmi/schema/Metadata.java
 e4e967da341ca824c7688b31c440b27d2247ab8e74abfefd584c28bfc99441f2  gencode/java/udmi/schema/Metrics.java
 189c32fc3b4b4d5a6f011b3ef8c7119278a19fbe490b5327c029ca471cfebed6  gencode/java/udmi/schema/Monitoring.java
 a43b73cbf5baadc6cd054f26aa97be58d3738a046acbba7c2ccedc2a21a1d7ac  gencode/java/udmi/schema/MonitoringMetric.java
@@ -219,7 +219,7 @@ e4cdab07942e52cae243b4dddbc72676794914947dd66df8add0f5969f31fd11  gencode/python
 72559779680c79dace3c5903022709852cbad7b77ed42af97debc8d852d61cb7  gencode/python/udmi/schema/events_validation.py
 a3ea178f98a4993da708e2ec41f5ede74aa92d742ffbe201a77adef803d4143a  gencode/python/udmi/schema/events_validation_device.py
 5660433b409bdaeacdce6b128660914586cca434e5373b3f94c04e89952642b4  gencode/python/udmi/schema/level.py
-9577d11c5110af169aafd0e083d115c849ce17dfdc2b6b23b7186c69c1c333db  gencode/python/udmi/schema/metadata.py
+cba94e8281d42450fc7590d738a3061f43a53ba8d81acef1edfaebdd0ee4d9e2  gencode/python/udmi/schema/metadata.py
 cd81df64d7db21ab06f84cdbd07a808a641612c7c1124f9f2d928a898c3c767e  gencode/python/udmi/schema/model_alarmset.py
 b80b7f580fd7527da0c2caff063218488cf5bbd091daa64a6a5f9a5d45b135c2  gencode/python/udmi/schema/model_alarmset_alarm.py
 6d19eb94063acbc8e5432b9827e8b20f4f8cd02efa6ae760fabe60652c69542d  gencode/python/udmi/schema/model_cloud.py

.github/workflows/testing.yml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
   pull_request:
   schedule:
-    - cron: '0 13 * * *'
+    - cron: '13 * * * *'
 
 concurrency:
   group: ci-${{ github.repository }}

bin/test_sites

@@ -38,6 +38,7 @@ function redact_files {
             -e 's-.*tests/sites/configs/-    REDACTED/sites/configs/-' \
             -e 's-^      File not found: /app/tests/sites/-    REDACTED/sites/-' \
             -e 's-^      File not found: .*udmi/tests/-    REDACTED/-' \
+             -e 's/([0-9]+ known properties: )("[^"]*"|, | )+/\1REDACTED/g' \
             $file
     done
 }
@@ -74,7 +75,7 @@ for subset in $subsets; do
     for device in $devices; do
         devicesout=$devicesdir/$device/out
         if [[ -d $devicesout ]]; then
-            find $devicesout -name \*.json | xargs -i bash -c "jq -e . {} >/dev/null 2>&1 && jq . {} | sponge {} || true"
+            find $devicesout -name \*.json | xargs -i bash -c "jq -e . {} >/dev/null 2>&1 && jq 'del(.generation)' {} | sponge {} || true"
             redact_files $devicesout/* $site_dir/registration_summary.json
             diffs=$(diff -rq $devicesout $devicesdir/$device/expected 2>&1) || true
             if [[ -n $diffs ]]; then

schema/metadata.json

@@ -26,6 +26,12 @@
       "description": "Original version of the UDMI schema for this file",
       "type": "string"
     },
+    "generation": {
+      "description": "RFC 3339 timestamp UTC the registrar started",
+      "type": "string",
+      "format": "date-time",
+      "examples": ["2019-01-17T14:02:29.364Z"]
+    },
     "hash": {
       "description": "DEPRECATED",
       "type": "string",

tests/sites/downgrade/devices/DWN-2/expected/errors.map

@@ -2,4 +2,4 @@ Exceptions for DWN-2
   While loading device metadata for DWN-2
     While converting strict to udmi.schema.Metadata
       While converting string/string to udmi.schema.Metadata
-        Unrecognized field "grumpy" (class udmi.schema.Metadata), not marked as ignorable (18 known properties: "upgraded_from", "gateway", "operation", "externals", "hash", "system", "discovery", "alarmset", "localnet", "structure", "cloud", "timestamp", "version", "relationships", "features", "testing", "pointset", "policy"]) at [Source: (String)"{ "cloud" : { "auth_type" : "RS256" }, "grumpy" : { "reason" : "This is an intentional schema violation to test the tools" }, "hash" : "da852d15", "localnet" : { "families" : { "vendor" : { "addr" : "0x65" } } }, "pointset" : { "exclude_units_from_config" : true, "points" : { "filter_alarm_pressure_status" : { "ref" : "BV11.present_value", "units" : "No-units" }, "filter_differential_pressure" : { "[truncated 667 chars]; line: 5, column: 15] (through reference chain: udmi.schema.Metadata["grumpy"])
+        Unrecognized field "grumpy" (class udmi.schema.Metadata), not marked as ignorable (19 known properties: REDACTED]) at [Source: (String)"{ "cloud" : { "auth_type" : "RS256" }, "grumpy" : { "reason" : "This is an intentional schema violation to test the tools" }, "hash" : "da852d15", "localnet" : { "families" : { "vendor" : { "addr" : "0x65" } } }, "pointset" : { "exclude_units_from_config" : true, "points" : { "filter_alarm_pressure_status" : { "ref" : "BV11.present_value", "units" : "No-units" }, "filter_differential_pressure" : { "[truncated 667 chars]; line: 5, column: 15] (through reference chain: udmi.schema.Metadata["grumpy"])

tests/sites/downgrade/devices/DWN-4/expected/errors.map

@@ -2,4 +2,4 @@ Exceptions for DWN-4
   While loading device metadata for DWN-4
     While converting strict to udmi.schema.Metadata
       While converting string/string to udmi.schema.Metadata
-        Unrecognized field "subsystem" (class udmi.schema.LocalnetModel), not marked as ignorable (3 known properties: "families", "networks", "parent"]) at [Source: (String)"{ "cloud" : { "auth_type" : "RS256" }, "hash" : "da852d15", "localnet" : { "subsystem" : { "vendor" : { "id" : "0x65" } } }, "pointset" : { "exclude_units_from_config" : true, "points" : { "filter_alarm_pressure_status" : { "ref" : "BV11.present_value", "units" : "No-units" }, "filter_differential_pressure" : { "units" : "Bars" }, "filter_differential_pressure_sensor" : { "ref" : "AV"[truncated 471 chars]; line: 7, column: 20] (through reference chain: udmi.schema.Metadata["localnet"]->udmi.schema.LocalnetModel["subsystem"])
+        Unrecognized field "subsystem" (class udmi.schema.LocalnetModel), not marked as ignorable (3 known properties: REDACTED]) at [Source: (String)"{ "cloud" : { "auth_type" : "RS256" }, "hash" : "da852d15", "localnet" : { "subsystem" : { "vendor" : { "id" : "0x65" } } }, "pointset" : { "exclude_units_from_config" : true, "points" : { "filter_alarm_pressure_status" : { "ref" : "BV11.present_value", "units" : "No-units" }, "filter_differential_pressure" : { "units" : "Bars" }, "filter_differential_pressure_sensor" : { "ref" : "AV"[truncated 471 chars]; line: 7, column: 20] (through reference chain: udmi.schema.Metadata["localnet"]->udmi.schema.LocalnetModel["subsystem"])

tests/sites/subsystem/devices/DEV-1/expected/errors.map

@@ -2,4 +2,4 @@ Exceptions for DEV-1
   While loading device metadata for DEV-1
     While converting strict to udmi.schema.Metadata
       While converting string/string to udmi.schema.Metadata
-        Unrecognized field "network_id" (class udmi.schema.FamilyLocalnetModel), not marked as ignorable (7 known properties: "parent_id", "addr", "family", "shadow_id", "vendor_ref", "network", "adjunct"]) at [Source: (String)"{ "gateway" : { "gateway_id" : "GW-1", "target" : { "family" : "modbus", "network_id" : "modbus_1" } }, "localnet" : { "families" : { "modbus" : { "addr" : "1" } } }, "system" : { "hardware" : { "make" : "BOS", "model" : "device" }, "software" : { "firmware" : "v1" } }, "timestamp" : "2020-05-01T13:39:07Z", "version" : "1.5.2" }"; line: 6, column: 23] (through reference chain: udmi.schema.Metadata["gateway"]->udmi.schema.GatewayModel["target"]->udmi.schema.FamilyLocalnetModel["network_id"])
+        Unrecognized field "network_id" (class udmi.schema.FamilyLocalnetModel), not marked as ignorable (7 known properties: REDACTED]) at [Source: (String)"{ "gateway" : { "gateway_id" : "GW-1", "target" : { "family" : "modbus", "network_id" : "modbus_1" } }, "localnet" : { "families" : { "modbus" : { "addr" : "1" } } }, "system" : { "hardware" : { "make" : "BOS", "model" : "device" }, "software" : { "firmware" : "v1" } }, "timestamp" : "2020-05-01T13:39:07Z", "version" : "1.5.2" }"; line: 6, column: 23] (through reference chain: udmi.schema.Metadata["gateway"]->udmi.schema.GatewayModel["target"]->udmi.schema.FamilyLocalnetModel["network_id"])

validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java

@@ -231,6 +231,7 @@ class LocalDevice implements SiteDevice {
   public void initialize() {
     prepareOutDir();
     populatePointUrls();
+    ifNotNullThen(metadata, m -> m.generation = JsonUtil.getDate(generation));
     ifTrueThen(deviceKind == DeviceKind.LOCAL && metadata != null, this::validateMetadata);
     configure();
   }

validator/src/test/java/com/google/daq/mqtt/registrar/LocalDeviceTest.java

@@ -47,6 +47,17 @@ public void getMergedMetadata() {
     assertNull("raw cloud detail is defined", catchToNull(() -> rawMetadata.cloud.detail));
   }
 
+  @Test
+  public void generationSet() {
+    String testGeneration = JsonUtil.isoConvert();
+    SiteModel siteModel = new SiteModel(EMPTY_DEFAULTS_SITE);
+    LocalDevice localDevice = new LocalDevice(siteModel, DEVICE_ID, SCHEMAS, testGeneration,
+        DeviceKind.SIMPLE);
+    localDevice.initialize();
+    assertEquals("metadata generation set", testGeneration,
+        JsonUtil.isoConvert(localDevice.getMetadata().generation));
+  }
+
   private LocalDevice getTestInstance(String sitePath) {
     SiteModel siteModel = new SiteModel(sitePath);
     assertTrue("test device exists: " + DEVICE_ID, siteModel.deviceExists(DEVICE_ID));

validator/src/test/java/com/google/daq/mqtt/registrar/RegistrarTest.java

@@ -195,6 +195,18 @@ public void transitiveUpdate() {
     assertEquals("Transitive devices", 3, transitiveCreates);
   }
 
+  @Test
+  public void generationTimestamp() {
+    List<MockAction> mockActions = executeRegistrarPopulated(ImmutableList.of());
+    List<MockAction> updateActions = filterActions(mockActions, UPDATE_DEVICE_ACTION);
+    updateActions.forEach(action -> {
+      CloudModel model = (CloudModel) action.data;
+      String udmiMetadata = model.metadata.get("udmi_metadata");
+      Metadata metadata = com.google.udmi.util.JsonUtil.fromString(Metadata.class, udmiMetadata);
+      assertTrue("metadata generation timestamp", metadata.generation != null);
+    });
+  }
+
   @Test
   public void basicUpdates() {
     List<MockAction> mockActions = executeRegistrarPopulated(ImmutableList.of());