JessyBarrette
commented
2 months ago Regarding 1. and 5. all those metadata variables are now only available as global attributes. ERDDAP is able to move global attributes to a dataset variable, this feature was not known by us when we initially generated the IOS datasets. However, both NAFC and BIO newer datasets are using this feature.
So I would say as long as those different metadata are made available via global attributes, we're all good. Here's an example of global attributes generated for the test file KSK1_20130703_20140703_0152m.ctd
{
"id": "tests/parsers_test_files/dfo/ios/shell/mooring/CTD/KSK1_20130703_20140703_0152m.ctd",
"agency": "IOS, Ocean Sciences Division, Sidney, BC",
"country": "Canada",
"project": "WCTSS-A",
"scientist": "Hannah C.",
"platform": "John P. Tully",
"start_time": "2013-07-03T02:00:01+00:00",
"end_time": "2014-07-03T15:00:01+00:00",
"time_increment": "0 0 30 0 0 ! (day hr min sec ms)",
"time_units": "Minutes",
"time_zero": "UTC 2013/07/03 00:00:00.000",
"number_of_records": "17547",
"data_description": "CTD",
"pad": "-99",
"number_of_channels": "8",
"instrument_type": "Sea-Bird SBE37",
"instrument_model": "Sea-Bird SBE37",
"instrument_serial_number": "SBE37 5306",
"instrument_depth": "152",
"instrument_remarks": " Instrument depth in metres.\n",
"geographic_area": "Douglas Channel system",
"station": "KSK1",
"event_number": "1",
"latitude": 53.48,
"longitude": -129.20855,
"water_depth": "372",
"deployment_mission": "2013-50",
"deployment_type": "Sub Surface",
"deployment_time_anchor_dropped": "UTC 2013/07/03 01:46:00.000",
"deployment_remarks": " 2 wheels, 2540 lbs.\n",
"recovery_mission": "2014-20",
"recovery_time_anchor_released": "UTC 2014/07/03 13:12:00.000",
"calibration": null,
"comments": {
"1": " World Class, KSK1-A at 152m.",
"2": " Data are from SBE37-5306",
"3": " Data missing from 2014/05/19 21:30 to 2014/05/29 01:00."
},
"remarks": "",
"history": {
"$TABLE: PROGRAMS": [
" ! Name Vers Date Time Recs In Recs Out\n",
" ! -------- ------ ---------- -------- --------- ---------\n",
" RCM_CNVT 1.1 2014/09/11 11:06:41 0 17547\n",
" HDREDIT2 3.0.2 2016/12/30 11:18:52 ? ?\n"
],
"$REMARKS": [
" -The following RCM_CNVT parameters were used.\n",
" Number of comment lines in RAW file: 0\n",
" -HEADER EDITS: 2016/12/30 11:18:52\n"
],
"ios_transform_history": "IOS Transform History:\n2024-05-01T13:14:05.296869 - Generate new variable from Record_Number -> record_number\n2024-05-01T13:14:05.299002 - Generate new variable from Temperature -> TEMPPR01\n2024-05-01T13:14:05.299862 - Generate new variable from Conductivity -> CNDCST01\n2024-05-01T13:14:05.300717 - Generate new variable from Pressure -> PRESPR01\n2024-05-01T13:14:05.302804 - Generate new variable from Pressure -> apply lambda x: -1*gsw.z_from_p(x,ds.attrs['latitude'])) -> depth\n2024-05-01T13:14:05.303941 - Generate new variable from Salinity -> PSALST01\n2024-05-01T13:14:05.304993 - Generate new variable from Speed:Sound -> SVELXXXX\n"
},
"headers": "{\n\"COMMENTS\": {\n\"1\": \" World Class, KSK1-A at 152m.\",\n\"2\": \" Data are from SBE37-5306\",\n\"3\": \" Data missing from 2014/05/19 21:30 to 2014/05/29 01:00.\"\n},\n\"FILE\": {\n\"START TIME\": \" UTC 2013/07/03 02:00:01.000\\n\",\n\"END TIME\": \" UTC 2014/07/03 15:00:01.000\\n\",\n\"TIME INCREMENT\": \" 0 0 30 0 0 ! (day hr min sec ms)\\n\",\n\"TIME UNITS\": \" Minutes\\n\",\n\"TIME ZERO\": \" UTC 2013/07/03 00:00:00.000\\n\",\n\"NUMBER OF RECORDS\": \" 17547\\n\",\n\"DATA DESCRIPTION\": \" CTD\\n\",\n\"FILE TYPE\": \" ASCII\\n\",\n\"DATA TYPE\": \" REAL*4\\n\",\n\"PAD\": \" -99\\n\",\n\"NUMBER OF CHANNELS\": \" 8\\n\",\n\"$TABLE: CHANNELS\": [\n\" ! No Name Units Minimum Maximum\\n\",\n\" !--- --------------- ------------ -------------- --------------\\n\",\n\" 1 Record_Number n/a 1 17547\\n\",\n\" 2 Date YYYY/MM/DD 0 365\\n\",\n\" 3 Time HH:MM:SS 0 0.99999\\n\",\n\" 4 Temperature 'deg C' 6.4797 14.2827\\n\",\n\" 5 Conductivity S/m 0.1189E-01 3.427\\n\",\n\" 6 Pressure decibar -0.94E-01 158.138\\n\",\n\" 7 Salinity PSS-78 0.724E-01 33.1394\\n\",\n\" 8 Speed:Sound m/s 1454.755 1482.262\\n\"\n],\n\"$TABLE: CHANNEL DETAIL\": [\n\" ! No Pad Start Width Format Type Decimal_Places\\n\",\n\" !--- ---- ----- ----- ---------- ---- --------------\\n\",\n\" 1 -99 ' ' 8 F R4 1\\n\",\n\" 2 -99 ' ' ' ' YYYY/MM/DD D ' '\\n\",\n\" 3 -99 ' ' ' ' HH:MM:SS T ' '\\n\",\n\" 4 -99 ' ' 9 f R4 4\\n\",\n\" 5 -99 ' ' 12 f R4 5\\n\",\n\" 6 -99 ' ' 9 f R4 3\\n\",\n\" 7 -99 ' ' 9 f R4 4\\n\",\n\" 8 -99 ' ' 9 f R4 3\\n\"\n]\n},\n\"ADMINISTRATION\": {\n\"AGENCY\": \" IOS, Ocean Sciences Division, Sidney, BC\\n\",\n\"COUNTRY\": \" Canada\\n\",\n\"PROJECT\": \" WCTSS-A\\n\",\n\"SCIENTIST\": \" Hannah C.\\n\",\n\"PLATFORM\": \" John P. Tully\\n\"\n},\n\"LOCATION\": {\n\"GEOGRAPHIC AREA\": \" Douglas Channel system\\n\",\n\"STATION\": \" KSK1\\n\",\n\"EVENT NUMBER\": \" 1\\n\",\n\"LATITUDE\": \" 53 28.80000 N ! (deg min)\\n\",\n\"LONGITUDE\": \" 129 12.51300 W ! (deg min)\\n\",\n\"WATER DEPTH\": \" 372\\n\"\n},\n\"DEPLOYMENT\": {\n\"MISSION\": \" 2013-50\\n\",\n\"TYPE\": \" Sub Surface\\n\",\n\"TIME ANCHOR DROPPED\": \" UTC 2013/07/03 01:46:00.000\\n\",\n\"$REMARKS\": [\n\" 2 wheels, 2540 lbs.\\n\"\n]\n},\n\"RECOVERY\": {\n\"MISSION\": \" 2014-20\\n\",\n\"TIME ANCHOR RELEASED\": \" UTC 2014/07/03 13:12:00.000\\n\"\n},\n\"INSTRUMENT\": {\n\"TYPE\": \" Sea-Bird SBE37\\n\",\n\"MODEL\": \" Sea-Bird SBE37\\n\",\n\"SERIAL NUMBER\": \" SBE37 5306\\n\",\n\"DEPTH\": \" 152\\n\",\n\"$REMARKS\": [\n\" Instrument depth in metres.\\n\"\n]\n},\n\"RAW\": {\n\"START TIME\": \" UTC 2013/07/03 02:00:01.000\\n\",\n\"FIRST GOOD REC TIME\": \" UTC 2013/07/03 02:00:01.000\\n\",\n\"END TIME\": \" UTC 2014/07/03 15:00:01.000\\n\",\n\"TIME INCREMENT\": \" 0 0 30 0 0 ! (day hr min sec ms)\\n\",\n\"NUMBER OF RECORDS\": \" 17547\\n\",\n\"VOLUME ID\": \" 5306-KSK1-\\n\",\n\"FIRST RECORD\": \" '6.9340, 3.34215, 155.900, 32.9421, 1478.381, 03'\\n\",\n\"CONTINUED\": \" ' Jul 2013, 02:00:01'\\n\",\n\"$TABLE: CHANNELS\": [\n\" ! Averaging (day hr min sec ms)\\n\",\n\" !Name Raw Units Interval Time Lag\\n\",\n\" !---- --------- --------- -----------\\n\"\n]\n},\n\"HISTORY\": {\n\"2\": \" $TABLE: PROGRAMS\",\n\"3\": \" ! Name Vers Date Time Recs In Recs Out\",\n\"4\": \" ! -------- ------ ---------- -------- --------- ---------\",\n\"5\": \" RCM_CNVT 1.1 2014/09/11 11:06:41 0 17547\",\n\"6\": \" HDREDIT2 3.0.2 2016/12/30 11:18:52 ? ?\",\n\"7\": \" $END\",\n\"8\": \" $REMARKS\",\n\"9\": \" -The following RCM_CNVT parameters were used.\",\n\"10\": \" Number of comment lines in RAW file: 0\",\n\"11\": \" -HEADER EDITS: 2016/12/30 11:18:52\",\n\"12\": \" $END\"\n}\n}",
"source": "tests/parsers_test_files/dfo/ios/shell/mooring/CTD/KSK1_20130703_20140703_0152m.ctd",
"ios_header_version": "2.0 ",
"ocean_data_transform_version": "0.3.0",
"product_version": "ios_header=2.0 ; ocean-data-transform=0.3.0",
"date_created": "2016-12-30T11:18:52.130000+00:00",
"institution": "DFO IOS",
"ices_edmo_code": 4155,
"sdn_institution_urn": "SDN:EDMO::4155",
"infoUrl": "https://science.gc.ca/site/science/en/educational-resources/marine-and-freshwater-sciences/institute-ocean-sciences",
"ioc_country_code": 18,
"naming_authority": "ca.gc.ios",
"iso_3166_country_code": "CA",
"platform_vocabulary": "https://vocab.nerc.ac.uk/search_nvs/C17/",
"instrument_vocabulary": "https://vocab.nerc.ac.uk/search_nvs/L22/",
"Conventions": "CF-1.6,CF-1.7,CF-1.8,ACDD1.1,ACDD-1.3,IOOS-1.2",
"standard_name_vocabulary": "CF Standard Name Table v78",
"creator_name": "Institute of Ocean Sciences (IOS)",
"creator_institution": "Institute of Ocean Sciences (IOS)",
"creator_email": "info@dfo-mpo.gc.ca",
"creator_country": "Canada",
"creator_sector": "gov_federal",
"creator_url": "info@dfo-mpo.gc.ca",
"creator_type": "institution",
"time_coverage_resolution": "P0DT0H0M0.0000018S",
"featureType": "timeSeries",
"cdm_data_type": "Timeseries"
}I can see that platform_depth is not available and can't really see an equivalent within the IOS files. I can see the *INSTRUMENT -> DEPTH within the header which I believe is equivalent. Can you confirm that @guanlu129?
Review of moored CTD data files conversion: For IOS moored CTD data, I’ve compared the netCDF file generated by ODPY and by the old ios_data_tranform, and would like to suggest a few modifications:
[ ] 1. In the ODPY version, missing instrument depth: float instrument_depth ; instrument_depth:_FillValue = NaNf ; instrument_depth:long_name = "Instrument Depth" ; instrument_depth:standard_name = "instrument_depth" ; instrument_depth:units = "m" ;
[x] 2. In the ODPY version, 1) keep the PRESPR01 (pressure) as original without converting to depth float PRESPR01(time) ; PRESPR01:_FillValue = NaNf ; PRESPR01:long_name = "Pressure" ; PRESPR01:standard_name = "sea_water_pressure" ; PRESPR01:units = "decibar" ; PRESPR01:featureType = "timeSeries" ;
[x] convert pressure to depth if there’s no depth channel in the ASCII file.
[x] 3. Missing long name and standard name for Time in ODPY version: double time(time) ; time:_FillValue = NaN ; time:long_name = "time" ; time:standard_name = "time" ; time:units = "seconds since 1970-01-01 00:00:00+0000"
[ ] #83
Question: do we really need these three variables in the file?
float sea_water_temperature(time) ; sea_water_temperature:_FillValue = NaNf ; sea_water_temperature:long_name = "Sea Water Temperature" ; sea_water_temperature:standard_name = "sea_water_temperature" ; sea_water_temperature:units = "degC" ; sea_water_temperature:comment = "{'2024-01-27T07:06:04.107038 UTC': 'Create sea_water_temperature variable and apply the following parameters: TEMPS901'}" ;
float sea_water_practical_salinity(time) ; sea_water_practical_salinity:_FillValue = NaNf ; sea_water_practical_salinity:long_name = "Sea Water Practical Salinity" ; sea_water_practical_salinity:standard_name = "sea_water_practical_salinity" ; sea_water_practical_salinity:units = "PSS-78" ; sea_water_practical_salinity:comment = "{\'2024-01-27T07:06:04.112308 UTC\': \'Create sea_water_practical_salinity variable and apply the following parameters: PSALST01\'}" ;
float sea_water_pressure(time) ; sea_water_pressure:_FillValue = NaNf ; sea_water_pressure:long_name = "Sea Water Pressure in dbar" ; sea_water_pressure:standard_name = "sea_water_pressure" ; sea_water_pressure:units = "dbar" ; sea_water_pressure:comment = "{\'2024-01-27T07:06:04.130787 UTC\': \'Create sea_water_pressure variable and apply the following parameters: PRESPR01\'}" ;
[ ] 5. In the old version, there are a few metadata were written in the data. Is it necessary to repeat info in data section? filename = "chat3_20220722_20230802_0015m_L1.ctd" ; country = "Canada" ; mission_id = "2023-069" ; scientist = "Hannah C." ; project = "BC-shelf 7" ; agency = "IOS, Ocean Sciences Division, Sidney, BC" ; platform = "John P. Tully" ; instrument_type = "SBE37SMP" ; instrument_model = "SBE37SMP" ; instrument_serial_number = "SBE37SMP-11725" ; instrument_depth = 15 ; latitude = 54.52715 ; longitude = -130.5415 ; geographic_area = "Hecate-Strait " ; event_number = "4" ; profile = "2023-069-0004" ;
[x] 6. Missing vocabulary for file_type=ctd; variable name=salinity:absolute,units=g/kg (TEOS-10): E.g. : juan2_20210606_20220503_0015m_L1.ctd