Update configuration for multiple nodes

Feature request

We need the configuration to handle multiple nodes attached to one receiver.

Here's an example of what the configuration should look like (see the comments for extra notes)

```js { "gateway": { "serial_buffer_rx_size": 4095, "serial_buffer_tx_size": 1280, "baudrate": 2300000, "endian": "little", "installation_reference": "ost-wt-evaluation", "longitude": "001", "latitude": "001", "turbine_id": "OST_WIND", "receiver_firmware_version": "2.1", "packet_key_offset": 245 }, "nodes": { "0": { "node_firmware_version": "whatever", "blade_id": "BAV01", "mics_freq": 15625, "mics_bm": 1023, "baros_freq": 100, "diff_baros_freq": 1000, "baros_bm": 1023, "acc_freq": 100, "acc_range": 16, "gyro_freq": 100, "gyro_range": 2000, "mag_freq": 12.5, "analog_freq": 16384, "constat_period": 45, "max_timestamp_slack": 0.005, "max_period_drift": 0.02, "type_handle_def": 255, "mics_samples_per_packet": 8, "imu_samples_per_packet": 40, "analog_samples_per_packet": 60, "baros_samples_per_packet": 1, "diff_baros_samples_per_packet": 24, "constat_samples_per_packet": 24, "sensor_names": [ "Mics", "Baros_P", "Baros_T", "Diff_Baros", "Acc", "Gyro", "Mag", "Analog Vbat", "Constat" ], "default_handles": { "34": "Abs. baros", "36": "Diff. baros", "38": "Mic 0", "40": "Mic 1", "42": "IMU Accel", "44": "IMU Gyro", "46": "IMU Magnetometer", "48": "Analog1", "50": "Analog2", "52": "Constat", "54": "Cmd Decline", "56": "Sleep State", "58": "Info Message" }, "decline_reason": { "0": "Bad block detection ongoing", "1": "Task already registered, cannot register again", "2": "Task is not registered, cannot de-register", "3": "Connection Parameter update unfinished" }, "sleep_state": { "0": "Exiting sleep", "1": "Entering sleep" }, "info_type": { "0": "Battery info" }, "samples_per_packet": { "Mics": 8, "Diff_Baros": 24, "Baros_P": 1, "Baros_T": 1, "Acc": 40, "Gyro": 40, "Mag": 40, "Analog Vbat": 60, "Constat": 24 }, "number_of_sensors": { "Mics": 10, "Baros_P": 40, "Baros_T": 40, "Diff_Baros": 5, "Acc": 3, "Gyro": 3, "Mag": 3, "Analog Vbat": 1, "Constat": 4 }, "sensor_conversion_constants": { "Mics": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], "Diff_Baros": [1, 1, 1, 1, 1], "Baros_P": [ 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96 ], "Baros_T": [ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100 ], "Acc": [1, 1, 1], "Gyro": [1, 1, 1], "Mag": [1, 1, 1], "Analog Vbat": [1], "Constat": [1, 1, 1, 1] }, "period": { "Mics": 6.4e-5, "Baros_P": 0.01, "Baros_T": 0.01, "Diff_Baros": 0.001, "Acc": 0.01, "Gyro": 0.01, "Mag": 0.08, "Analog Vbat": 6.103515625e-5, "Constat": 0.045 }, "sensor_commands": { "start": ["startBaros", "startDiffBaros", "startIMU", "startMics"], "stop": ["stopBaros", "stopDiffBaros", "stopIMU", "stopMics"], "configuration": [ "configBaros", "configAccel", "configGyro", "configMics" ], "utilities": [ "getBattery", "setConnInterval", "tpcBoostIncrease", "tpcBoostDecrease", "tpcBoostHeapMemThr1", "tpcBoostHeapMemThr2", "tpcBoostHeapMemThr4" ] }, "sensor_coordinates": { "Mics": [ [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0] ], "Baros_P": [ [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0] ], "Baros_T": [ [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0] ], "Diff_Baros": [ [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0] ], "Acc": [ [0, 0, 0], [0, 0, 0], [0, 0, 0] ], "Gyro": [ [0, 0, 0], [0, 0, 0], [0, 0, 0] ], "Mag": [ [0, 0, 0], [0, 0, 0], [0, 0, 0] ], "Analog Vbat": [[0, 0, 0]], "Constat": [ [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0] ] } }, "1": {}, "2": {} }, "session": { "label": null } } ```

Here's what the configuration looks like now

``` { "mics_freq": 15625, "mics_bm": 1023, "baros_freq": 100, "diff_baros_freq": 1000, "baros_bm": 1023, "acc_freq": 100, "acc_range": 16, "gyro_freq": 100, "gyro_range": 2000, "mag_freq": 12.5, "analog_freq": 16384, "constat_period": 45, "serial_buffer_rx_size": 4095, "serial_buffer_tx_size": 1280, "baudrate": 2300000, "endian": "little", "max_timestamp_slack": 0.005, "max_period_drift": 0.02, "packet_key": 245, "type_handle_def": 255, "mics_samples_per_packet": 8, "imu_samples_per_packet": 40, "analog_samples_per_packet": 60, "baros_samples_per_packet": 1, "diff_baros_samples_per_packet": 24, "constat_samples_per_packet": 24, "sensor_names": [ "Mics", "Baros_P", "Baros_T", "Diff_Baros", "Acc", "Gyro", "Mag", "Analog Vbat", "Constat" ], "default_handles": { "34": "Abs. baros", "36": "Diff. baros", "38": "Mic 0", "40": "Mic 1", "42": "IMU Accel", "44": "IMU Gyro", "46": "IMU Magnetometer", "48": "Analog1", "50": "Analog2", "52": "Constat", "54": "Cmd Decline", "56": "Sleep State", "58": "Info Message" }, "decline_reason": { "0": "Bad block detection ongoing", "1": "Task already registered, cannot register again", "2": "Task is not registered, cannot de-register", "3": "Connection Parameter update unfinished" }, "sleep_state": { "0": "Exiting sleep", "1": "Entering sleep" }, "info_type": { "0": "Battery info" }, "samples_per_packet": { "Mics": 8, "Diff_Baros": 24, "Baros_P": 1, "Baros_T": 1, "Acc": 40, "Gyro": 40, "Mag": 40, "Analog Vbat": 60, "Constat": 24 }, "number_of_sensors": { "Mics": 10, "Baros_P": 40, "Baros_T": 40, "Diff_Baros": 5, "Acc": 3, "Gyro": 3, "Mag": 3, "Analog Vbat": 1, "Constat": 4 }, "sensor_conversion_constants": { "Mics": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "Diff_Baros": [ 1, 1, 1, 1, 1 ], "Baros_P": [ 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96, 40.96 ], "Baros_T": [ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100 ], "Acc": [ 1, 1, 1 ], "Gyro": [ 1, 1, 1 ], "Mag": [ 1, 1, 1 ], "Analog Vbat": [ 1 ], "Constat": [ 1, 1, 1, 1 ] }, "period": { "Mics": 6.4e-05, "Baros_P": 0.01, "Baros_T": 0.01, "Diff_Baros": 0.001, "Acc": 0.01, "Gyro": 0.01, "Mag": 0.08, "Analog Vbat": 6.103515625e-05, "Constat": 0.045 }, "sensor_commands": { "start": ["startBaros", "startDiffBaros", "startIMU", "startMics"], "stop": ["stopBaros", "stopDiffBaros", "stopIMU", "stopMics"], "configuration": ["configBaros", "configAccel", "configGyro", "configMics"], "utilities": [ "getBattery", "setConnInterval", "tpcBoostIncrease", "tpcBoostDecrease", "tpcBoostHeapMemThr1", "tpcBoostHeapMemThr2", "tpcBoostHeapMemThr4" ] }, "installation_data": { "installation_reference": "ost-wt-evaluation", "longitude": "001", "latitude": "001", "turbine_id": "OST_WIND", "blade_id": "BAV01", "hardware_version": "2.1", "sensor_coordinates": { "Mics": [ [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ] ], "Baros_P": [ [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ] ], "Baros_T": [ [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ] ], "Diff_Baros": [ [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ] ], "Acc": [ [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ] ], "Gyro": [ [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ] ], "Mag": [ [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ] ], "Analog Vbat": [ [ 0, 0, 0 ] ], "Constat": [ [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ], [ 0, 0, 0 ] ] } }, "session_data": { "label": null } } ```

Change Notes

gateway-specific values are split out from node-specific values
node-specific values are repeated, one set for each node, keyed on an integer.
- they are keyed rather than put in an array so that, in future, we could change them to strings or hex or whatever
session_data is renamed session because _data is redundant/obvious
gateway.receiver_firmware_version was previously called hardware_version but that was ambiguous
nodes.x.node_firmware_version is the other part of disambiguating hardware_version.
blade_id was part of data that ended up in the installations table (as was sensor position data). That installations table will require refactoring in the db / cloud functions because it shouldn't have that node specific data on it, we should just keep that in configurations and query through for it.
much of the configuration is duplicated between the nodes, but that is done so that in future, it'll be easier to run different firmware versions on the same installation.
packet_key (which was 254 for the v2 firmware) has been replaced with packet_key_offset (see below for how to find the packet key).

Steps

[x] Update Configuration class to accept that new form input.
[x] Identify the set of possible packet keys and use them to identify packets being received in PacketReader.read_packets() (instead of one fixed packet key being used as the id). Subtract the packet_key_offset to get the node_id of the packet, and include that in the packet queue data.
[x] Adapt the persistence layer to allow for different nodes, and different nodes having different sensor names (perhaps by dividing persistence by node?)
[x] Update all usage of self.config to use the appropriate variables (now deeper in the config) (see " Using node-specific data in the packet parser" below).
[x] Update the database schema to:
- [x] Add node_id to the measurements table so that each measurement is referenced to the individual table.
- [x] Remove the geometry field from the installations table because it is duplicated from the configurations table.
- See these notes on how to delete columns
- The reason WHY we did this was so that we could show where the sensor locations were on the blade in the list of installations. But I think we should do this more smartly through materialized views rather than duping that data, particularly since it could change over time as configurations change.
- [x] Remove the blade_id value from the installations table (now node-specific, and again duplicated from the configurations table)
- [x] Update the sensor data sent to the cloud to include node_id.
- [x] Update the cloud function to correctly parse data from the configuration.
- [x] Update create_installation and add_sensor_type fo rhte new configuration pattern (the create_installation will also have to respect the new DB schema)

Getting `node_id` from `packet_key_offset`

The node id is encoded in, but is not the same as, the packet key. The encoding is as: packet_key = node_id + packet_key_offset

Thus, for each node, packet_key = config.gateway.packet_key_offset + node_id.

BACKWARD COMPATIBILITY WITH V2 is achievable by setting the v2 up with a node_id=0 and thus the packet key is trivially equal to the offset.

Using node-specific data in the packet parser

Use something like:

        previous_timestamp = {}
        data = {}
        for node_id in self.config.node_ids: # todo add property to config
            data[node_id]={}
            previous_timestamp[node_id] = {}
            for sensor_name in self.config.nodes[node_id].sensor_names:
                previous_timestamp[node_id][sensor_name] = -1
                data[node_id][sensor_name] = [
                    ([0] * self.config.samples_per_packet[sensor_name])
                    for _ in range(self.config.number_of_sensors[sensor_name])
                ]

Extra Notes

[ ] The getBattery firmware bug (if you send getBattery you had to reset the sensor after) is apparently fixed in v3.

aerosense-ai / data-gateway