MathewBiddle
commented
1 year ago Need to check my code against @kbailey-noaa
Mine:
import pandas as pd
df_glider = pd.read_csv('https://gliders.ioos.us/erddap/tabledap/allDatasets.csvp?minTime%2CmaxTime%2CdatasetID')
df_glider.dropna(
axis=0,
inplace=True,
)
# drop delayed datasets
df_glider = df_glider[df_glider["datasetID"].str.contains("delayed")==False]
df_glider[['minTime (UTC)','maxTime (UTC)']] = df_glider[
['minTime (UTC)','maxTime (UTC)']
].apply(pd.to_datetime)
start_date = '2000-01-01'
end_date = '2022-12-31'
# find glider deployments between 10/01 and 12/31
glider_day_within = df_glider.loc[
(df_glider['minTime (UTC)'] > pd.to_datetime(start_date,utc=True)) &
(df_glider['maxTime (UTC)'] < pd.to_datetime(end_date,utc=True))
]
# gliders that start before 10/01 and end after 12/31
glider_day_outside = df_glider.loc[
(df_glider['minTime (UTC)'] < pd.to_datetime(start_date,utc=True)) &
(df_glider['maxTime (UTC)'] > pd.to_datetime(end_date,utc=True))
]
glider_day_outside.loc[:, 'maxTime (UTC)'] = pd.to_datetime(end_date, utc=True)
glider_day_outside.loc[:, 'minTime (UTC)'] = pd.to_datetime(start_date, utc=True)
# drop the ones from above as they will be duplicates in the next round of filtering
df_glider.drop(axis=0, index=glider_day_outside.index, inplace=True)
# Find gliders that start before 10/01 and end after 10/01
glider_day_lower = df_glider.loc[
(df_glider['minTime (UTC)'] < pd.to_datetime(start_date,utc=True)) &
(df_glider['maxTime (UTC)'] > pd.to_datetime(start_date,utc=True))
]
glider_day_lower.loc[:,'minTime (UTC)'] = pd.to_datetime(start_date, utc=True)
# Find gliders that start before 12/31 and end after 12/31.
glider_day_upper = df_glider.loc[
(df_glider['minTime (UTC)']<pd.to_datetime(end_date,utc=True)) &
(df_glider['maxTime (UTC)']>pd.to_datetime(end_date,utc=True))
]
glider_day_upper.loc[:,'maxTime (UTC)'] = pd.to_datetime(end_date, utc=True)
# Combine it all together into one DF.
glider_subset = pd.concat([glider_day_lower,
glider_day_within,
glider_day_upper,
glider_day_outside],
verify_integrity=True)
# Calculate the days between min time and max time.
glider_subset['glider_days'] = (glider_subset['maxTime (UTC)'] - glider_subset['minTime (UTC)']).dt.days
# Calculate total glider days.
glider_days = glider_subset['glider_days'].sum()
print("Glider days between %s and %s: %s" % (start_date,end_date,glider_days))Kathy's:
# import ipydatetime
# from datetime import datetime
# from ipywidgets import VBox, HBox, Label, BoundedFloatText
# dt0 = ipydatetime.NaiveDatetimePicker(value=datetime(2000, 1, 1, 0, 0, 0), description='start time:')
# dt1 = ipydatetime.NaiveDatetimePicker(value=datetime(2022, 12, 31, 23, 59, 59), description='end time:')
# south = BoundedFloatText(
# value=-90,
# min=-90,
# max=90,
# description='min lat:',
# )
# north = BoundedFloatText(
# value=90,
# min=-90,
# max=90,
# description='max lat:',
# )
# west = BoundedFloatText(
# value=-180,
# min=-180,
# max=180,
# description='min lon:',
# )
# east = BoundedFloatText(
# value=180,
# min=-180,
# max=180,
# description='max lon:',
# )
# from ipywidgets import Layout, Button, Box
# box_layout = Layout(
# display="flex",
# flex_flow="column",
# align_items="center"
# )
# items = [
# VBox([dt0, dt1]),
# north,
# HBox([west, east]),
# south,
# ]
# box = Box(children=items, layout=box_layout)
# box
# dt0 = dt0.value
# dt1 = dt1.value
# south = south.value
# north = north.value
# west = west.value
# east = east.value
# params = {
# "min_time": dt0,
# "max_time": dt1,
# "min_lat": south,
# "max_lat": north,
# "min_lon": west,
# "max_lon": east,
# }
# from gdutils import GdacClient
# client = GdacClient()
# client.search_datasets(params=params, include_delayed_mode=False)
# client.datasets
# # Count the total number of deployments within the dt0:dt1 time window
# num_deployments = client.datasets.shape[0]
# # Count the number of glider days within the dt0:dt1 time window
# glider_days = client.glider_days_per_yyyymmdd.loc[dt0:dt1].sum()
# # count the number of profiles per dataset
# profile_count = client.profiles_per_yyyymmdd.loc[dt0:dt1].sum()
# datasets = client.datasets.copy()
# import warnings
# sea_names = []
# funding_sources = []
# for dataset_id, row in datasets.iterrows():
# # Fetch the dataset description from ERDDAP
# info = client.get_dataset_metadata(dataset_id)
# if info.empty:
# continue
# # Find all global NetCDF attributes
# nc_globals = info.loc[info["Variable Name"] == "NC_GLOBAL"]
# # Find the sea_name global attribute
# sea_name_attr = nc_globals.loc[nc_globals["Attribute Name"] == "sea_name"]
# sea_name = "unknown"
# if not sea_name_attr.empty:
# sea_name = sea_name_attr.Value.iloc[0]
# sea_name = sea_name or "unknown"
# else:
# warnings.warn(f"{dataset_id}: sea_name NC_GLOBAL not found")
# # Find all global attributes that begin with "acknowledg" as this attribute typically contains the funding sources
# funding_attr = nc_globals.loc[nc_globals["Attribute Name"].str.startswith("acknowledg")]
# funding = "unknown"
# if not funding_attr.empty:
# funding = funding_attr.Value.iloc[0]
# funding = funding or "unknown"
# else:
# warnings.warn(f"{dataset_id}: acknowledgment NC_GLOBAL not found")
# sea_names.append(sea_name)
# funding_sources.append(funding)
# # Count only the days and profiles that are dt0:dt1 inclusive
# days = client.datasets_days[dataset_id].loc[dt0:dt1].sum()
# profiles = client.datasets_profiles[dataset_id].loc[dt0:dt1].sum()
# datasets.loc[dataset_id, ("days", "num_profiles")] = [days, profiles]
# print(f"time ranged from {dt0} to {dt1}")
# print(f"""
# {num_deployments = }
# {glider_days = }
# {profile_count = }
# """
# )
ocefpaf
Similar to the GTS calculations, it would be useful to have gliderDAC quarterly metrics too.
https://github.com/MathewBiddle/ioos_by_the_numbers/tree/main/gts