sean-rohan-NOAA commented 2 years ago

Automated processing end-to-end test.

Install the package:

remotes::install_github("sean-rohan-NOAA/gapctd")

Run the code:

library(gapctd)
library(ggplot2)
library(sf)

vessel <- 94
year <- 2021
region <- "BS"

# Establish Oracle connection connection
channel <- gapctd::get_connected()

try_methods <- c("sbe19plus_v0", 
                 "sbe19plus_v1",
                 "sbe19plus_v2",
                 "sbe19plus_v3")

# First CTD
for(hh in 1:length(try_methods)) {
  gapctd::run_method(vessel = vessel,
                     year = year,
                     region = region,
                     ctd_dir = "G:/RACE_CTD/data/2021/ebs/v94_ctd1",
                     channel = channel,
                     processing_method = try_methods[hh],
                     rm_files = TRUE)
}

# Compare results of processing methods and assign best methods
dat_binned <- gapctd::compare_methods(prefix = "/binavg/align_ctd_method",
                                      processing_method = try_methods,
                                      return_output = TRUE,
                                      pattern_downcast = "downcast_binavg.cnv",
                                      pattern_upcast = "upcast_binavg.cnv")

saveRDS(object = dat_binned, file = here::here("output", "compare_methods.rds"))

# Move best cast files to /final_cnv/
gapctd::move_to_final_cnv(best_df = dat_binned$best_df)

# Density inversion test and correction
gapctd::correct_density_inversion(threshold = -1e-5, threshold_method = "bv", correct_inversion = TRUE)

sean-rohan-NOAA commented 2 years ago

Test case 2 - Automated align and ctm parameter estimation

library(gapctd)

vessel <- 94
year <- 2021
region <- "BS"
try_method <- "sbe19plus_v0"
ctd_dir <- "G:/RACE_CTD/data/2021/ebs/v94_ctd1"
channel <- gapctd::get_connected(schema = "AFSC")

# Run using 'typical' parameters
gapctd::run_method(vessel = vessel,
                   year = year,
                   region = region,
                   channel = channel,
                   processing_method = "sbe19plus_v2",
                   storage_directory = here::here("output", "sbe19plus_v2"),
                   ctd_dir = ctd_dir)

# Process without alignment or cell thermal mass correction
gapctd::run_method(vessel = vessel,
                   year = year,
                   region = region,
                   channel = channel,
                   processing_method = try_method,
                   storage_directory = here::here("output", try_method),
                   ctd_dir = ctd_dir)

# Estimate alignment module parameters for temperature
alignment_df <-
  gapctd::run_alignment_calcs(
    profile_files = sort(c(
      list.files(
        here::here("output", try_method),
        full.names = TRUE,
        pattern = "downcast.cnv"
      ),
      list.files(
        here::here("output", try_method),
        full.names = TRUE,
        pattern = "upcast.cnv"
      )
    )),
    make_diagnostic_plots = TRUE,
    min_pressure = 4,
    cor_method = "spearman",
  )

# Save alignment parameters to .rds
saveRDS(object = alignment_df, here::here("output", paste0(region, "_", year, "_", vessel, "_align_pars.rds")))

# Align profile data
gapctd::run_method(vessel = vessel,
                   year = year,
                   region = region,
                   channel = channel,
                   processing_method = try_method,
                   storage_directory = here::here("output", paste0(try_method, "_align")),
                   ctd_dir = ctd_dir,
                   alignment_df = readRDS(file = here::here("output", paste0(region, "_", year, "_", vessel, "_align_pars.rds"))))

# Estimate cell thermal mass (CTM) correction module parameters for aligned data
ctm_df <- gapctd::run_ctm_adjust_tsarea(profile_files = sort(c(list.files(here::here("output", "sbe19plus_v0_align"),
                                                                                 full.names = TRUE, pattern = "downcast.cnv"),
                                                                      list.files(here::here("output", "sbe19plus_v0_align"),
                                                                                 full.names = TRUE, pattern = "upcast.cnv"))),
                                               min_pressure = 4,
                                               optim_method = "BFGS",  #"SANN",
                                               optim_maxit = 500,
                                               start_alpha = 0.04,
                                               start_tau = 8)

# Save CTM correction parameters to .rds
saveRDS(ctm_df, file = here::here("output", paste0(region, "_", year, "_", vessel, "_ctm_area_pars.rds")))

# Apply cell thermal mass correction
gapctd::run_method(vessel = vessel,
                   year = year,
                   region = region,
                   channel = channel,
                   processing_method = try_method,
                   storage_directory = here::here("output", paste0(try_method, "_ctm_area")),
                   ctd_dir = ctd_dir,
                   ctm_df = readRDS(file = here::here("output", paste0(region, "_", year, "_", vessel, "_ctm_area_pars.rds"))))

# Compare profiles based on the area between salinity profiles
compare_df <- gapctd::compare_methods(prefix = "/binavg/align_ctd_method",
                                      method_labels = c("Estimated (Raw)", "Estimated (Aligned)", "Estimated (CTM-Area)", "Typical method"),
                                      processing_method = c("sbe19plus_v0", "sbe19plus_v0_align", "sbe19plus_v0_ctm_area", "sbe19plus_v2"),
                                      return_output = TRUE,
                                      min_pressure_bin = 4,
                                      pattern_downcast = "downcast_binavg.cnv",
                                      pattern_upcast = "upcast_binavg.cnv")

coleary-noaa commented 2 years ago

Baseline test case with BS data worked succesfully. My only comment is probably worth adding text to the readme about having to click through the SBEData processing text boxes after some of the run_method() functions, it's not super obvious which buttons to push. Will test on a GOA data subset next.

sean-rohan-NOAA commented 2 years ago

Thanks, Cecilia. I added error documentation here for now: https://github.com/afsc-gap-products/gapctd/blob/main/doc/SBE_data_processing_errors.md

coleary-noaa commented 2 years ago

Testing on GOA data

Reminders: -make sure .xlmcon file is named with "_DeployYEAR.xmlcon" appended at the end (e.g., "19-8104_Deploy2021.xmlcon" works and "19-810.xlmcon" does not work in batch file processing)

Changes made to code to work with GOA data too:

marked_event does not work for GOA files because there are less of those than other haul events, removed from innerjoin when creating haul events.
added in a line to run_method() make output directory that was not previously there, wasn't creating it on my computer with new data.
added some if statements for listing out files in the instances they don't exist (e.g. bad stations)

Retested on BS data after changes made and seems to still work, might want to check on your system too.

sean-rohan-NOAA commented 2 years ago

Reminders: -make sure .xlmcon file is named with "_DeployYEAR.xmlcon" appended at the end (e.g., "19-8104_Deploy2021.xmlcon" works and "19-810.xlmcon" does not work in batch file processing)

@coleary-noaa Can you provide a bit more detail about this issue? I didn't think it mattered what the config (.xmlcon) file is called as long as the filename extension is correct and the config file settings match the settings of the config that was used when the CTD was deployed. In the batch processing routines, the .xmlcon file in the source directory is copied to /psa_xmlcon/ then automatically detected and passed as an argument to SBE Data Processing via command line (system() function):

https://github.com/afsc-gap-products/gapctd/blob/acd4746906bba5455ecbf57350e965e5328d0bc6/R/setup_ctd_processing_dir.R#L76-L84

https://github.com/afsc-gap-products/gapctd/blob/acd4746906bba5455ecbf57350e965e5328d0bc6/R/run_sbe_batch.R#L40-L51

https://github.com/afsc-gap-products/gapctd/blob/acd4746906bba5455ecbf57350e965e5328d0bc6/R/run_sbe_batch.R#L88

https://github.com/afsc-gap-products/gapctd/blob/acd4746906bba5455ecbf57350e965e5328d0bc6/inst/extdata/SBE19plus_v0/sbe19plus_getdata.bat#L1-L4

In the EBS, there was one vessel where the config file wasn't correct even though the serial number matched. The issue seemed to be that the config file was the old file from when the CTDs were setup in the vertical deployment configuration and included extra channels for oxygen and pH, whereas the proper file was the config file for the horizontal configuration with only temperature, conductivity, and pressure (included Deploy in the name). I could definitely be wrong though since swapping the config files also meant the replacement config file had the same name as the config file that was used during deployment. But I recall there was an error message that made me think the name wasn't the problem.

There was another issue where multiple CTDs were used on one vessel but all of the files were in the same directory. I solved that problem by moving each unit to its own directory.

sean-rohan-NOAA commented 2 years ago

Retested on BS data after changes made and seems to still work, might want to check on your system too.

Yep, still works!

coleary-noaa commented 2 years ago

@coleary-noaa Can you provide a bit more detail about this issue? I didn't think it mattered what the config (.xmlcon) file is called as long as the filename extension is correct and the config file settings match the settings of the config that was used when the CTD was deployed. In the batch processing routines, the .xmlcon file in the source directory is copied to /psa_xmlcon/ then automatically detected and passed as an argument to SBE Data Processing via command line (system() function):

I just tried to replicate the issue and had no problems with any .xmlcon file regardless of how it was named as long as it matched the config file, so you're right. Guess it was just a fluke error. The first time I tried it it just said it couldn't find any .xmlcon file ( "run_sbe_batch: No .xmlcon file found in ", getwd(), "/psa_xmlcon/. Must have a valid .xmlcon file.") error ) and when I switched it from "19-8104.xmlcon" to "19-8104_Deploy2021.xmlcon" the error went away. Now it works with either, so perhaps it was related to some of the other issues I fixed or just a bug the first time I ran it. Either way, not an issue now so ignore!

sean-rohan-NOAA commented 2 years ago

Phew! Thanks for checking.

afsc-gap-products / gapctd

Test case #32

Test case 2 - Automated align and ctm parameter estimation