matschaffer
commented
6 years ago Should be able to use this as a list of what the TSV columns should be
{
"ingest-measurements-2018-08-16": {
"mappings": {
"_default_": {
"_meta": {
"version": "1.0.0"
},
"_all": {
"norms": false
},
"dynamic_templates": [
{
"strings_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
],
"date_detection": false,
"properties": {
"@timestamp": {
"type": "date"
},
"ingest": {
"properties": {
"location": {
"type": "geo_point",
"ignore_malformed": true
}
}
},
"pms_std01_0": {
"type": "float"
},
"service_uploaded": {
"type": "date"
}
}
},
"measurement": {
"_meta": {
"version": "1.0.0"
},
"_all": {
"norms": false
},
"dynamic_templates": [
{
"strings_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
],
"date_detection": false,
"properties": {
"@timestamp": {
"type": "date"
},
"bat_charge": {
"type": "float"
},
"bat_current": {
"type": "float"
},
"bat_voltage": {
"type": "float"
},
"dev_cfgdev": {
"type": "keyword",
"ignore_above": 1024
},
"dev_cfgsvc": {
"type": "keyword",
"ignore_above": 1024
},
"dev_cfgttn": {
"type": "keyword",
"ignore_above": 1024
},
"dev_comms_ant_fails": {
"type": "long"
},
"dev_comms_failures": {
"type": "long"
},
"dev_comms_power_fails": {
"type": "long"
},
"dev_comms_resets": {
"type": "long"
},
"dev_cpsi": {
"type": "keyword",
"ignore_above": 1024
},
"dev_dfu": {
"type": "keyword",
"ignore_above": 1024
},
"dev_err_con_data": {
"type": "long"
},
"dev_err_con_fona": {
"type": "long"
},
"dev_err_con_gateway": {
"type": "long"
},
"dev_err_con_wireless": {
"type": "long"
},
"dev_err_fona": {
"type": "long"
},
"dev_err_max01": {
"type": "long"
},
"dev_err_opc": {
"type": "long"
},
"dev_err_pms": {
"type": "long"
},
"dev_err_twi": {
"type": "long"
},
"dev_err_twi_info": {
"type": "keyword",
"ignore_above": 1024
},
"dev_firmware": {
"type": "keyword",
"ignore_above": 1024
},
"dev_free_memory": {
"type": "long"
},
"dev_humid": {
"type": "float"
},
"dev_iccid": {
"type": "keyword",
"ignore_above": 1024
},
"dev_last_failure": {
"type": "keyword",
"ignore_above": 1024
},
"dev_module_fona": {
"type": "keyword",
"ignore_above": 1024
},
"dev_module_lora": {
"type": "keyword",
"ignore_above": 1024
},
"dev_motion": {
"type": "boolean"
},
"dev_motion_events": {
"type": "long"
},
"dev_ntp_count": {
"type": "long"
},
"dev_oneshot_seconds": {
"type": "long"
},
"dev_oneshots": {
"type": "long"
},
"dev_press": {
"type": "float"
},
"dev_received_bytes": {
"type": "long"
},
"dev_restarts": {
"type": "long"
},
"dev_temp": {
"type": "float"
},
"dev_test": {
"type": "boolean"
},
"dev_transmitted_bytes": {
"type": "long"
},
"dev_uptime": {
"type": "long"
},
"device": {
"type": "long"
},
"device_urn": {
"type": "keyword",
"ignore_above": 1024
},
"env_humid": {
"type": "float"
},
"env_press": {
"type": "float"
},
"env_temp": {
"type": "float"
},
"gateway_lora_snr": {
"type": "long"
},
"gateway_received": {
"type": "keyword",
"ignore_above": 1024
},
"ingest": {
"properties": {
"location": {
"type": "geo_point",
"ignore_malformed": true
}
}
},
"lnd_7128ec": {
"type": "long"
},
"lnd_712u": {
"type": "long"
},
"lnd_7318c": {
"type": "long"
},
"lnd_7318u": {
"type": "long"
},
"lnd_78017w": {
"type": "long"
},
"loc_alt": {
"type": "long"
},
"loc_lat": {
"type": "float"
},
"loc_lon": {
"type": "float"
},
"loc_olc": {
"type": "keyword",
"ignore_above": 1024
},
"opc_c00_38": {
"type": "long"
},
"opc_c00_54": {
"type": "long"
},
"opc_c01_00": {
"type": "long"
},
"opc_c02_10": {
"type": "long"
},
"opc_c05_00": {
"type": "long"
},
"opc_c10_00": {
"type": "long"
},
"opc_csecs": {
"type": "long"
},
"opc_pm01_0": {
"type": "float"
},
"opc_pm02_5": {
"type": "float"
},
"opc_pm10_0": {
"type": "float"
},
"pms_c00_30": {
"type": "long"
},
"pms_c00_50": {
"type": "long"
},
"pms_c01_00": {
"type": "long"
},
"pms_c02_50": {
"type": "long"
},
"pms_c05_00": {
"type": "long"
},
"pms_c10_00": {
"type": "long"
},
"pms_csecs": {
"type": "long"
},
"pms_pm01_0": {
"type": "long"
},
"pms_pm02_5": {
"type": "long"
},
"pms_pm10_0": {
"type": "long"
},
"pms_std01_0": {
"type": "float"
},
"service_handler": {
"type": "keyword",
"ignore_above": 1024
},
"service_md5": {
"type": "keyword",
"ignore_above": 1024
},
"service_transport": {
"type": "keyword",
"ignore_above": 1024
},
"service_uploaded": {
"type": "date"
},
"when_captured": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
}
}
For http://edf.org/
They have a system we'd like to get our data into.
If we can provide hourly TSV on s3 similar to how we provide NDJSON today, they should be able to consume it w/o much work on their end.