lec170
commented
5 years ago Update:
I have been able to modify the configuration code properly. The validateConfiguration(config) does provide a positive result at this point. However, when I try to generate the analogs, my R session begins computing and then aborts when it is computing similarity matrices. Code below. Picture of the Rsession and the point it aborts is attached.
Here is the code below:
# Load libraries, source functions file
library(ncdf4); library(RAnEn)
source('~/geolab/projects/AnalogsSpatial/code/SSE_functions.R', echo=TRUE)
# Load Data
# Forecasts
nc.fcst.file <- '~/geolab_storage_V3/data/Analogs/ECMWF_Italy/ItalyFcst20180604.nc'
nc.fcst <- nc_open(nc.fcst.file)
# Capture a small subset of the data
fcst <- ncvar_get(nc.fcst, 'Data',start = c(1,1,1,1), count = c(5,2499,1095,16))
# "10U","10V","2T","2DPT","MSLP"
lon <- ncvar_get(nc.fcst, "Lon")[1:dim(fcst)[2]]
lat <- ncvar_get(nc.fcst, "Lat")[1:dim(fcst)[2]]
# Convert u and v components into wind speed and wind direction
fcst.calc <- fcst
u.10.fcst <- fcst[1,,,]
v.10.fcst <- fcst[2,,,]
dir.10.fcst <- UVtoDir(u.10.fcst, v.10.fcst)
spd.10.fcst <- UVtoSpd(u.10.fcst, v.10.fcst)
# Replace
fcst.calc[1,,,] <- dir.10.fcst
fcst.calc[2,,,] <- spd.10.fcst
# "10WD","10WS","2T","2DPT","MSLP"
fcst.times <- ncvar_get(nc.fcst, "Times") # range(as.POSIXct(times,origin="1970-01-01",tz="UTC"))
flts <- ncvar_get(nc.fcst, "FLT",start = 1, count = 16) # The real time
# New: we should keep from 1 to 2.
fcst.flt.to.keep <- c(seq(from = 1, to = dim(fcst)[4], by = 2)) # You can save time and memory by skipping and not feeding all the data. It can handle the missing FLT in the program. However, we subset bc it saves time.
fcst.aligned <- fcst.calc[,,,fcst.flt.to.keep]
flts.subset <- flts[fcst.flt.to.keep]
# Observations (Analysis Fields)
nc.analy.file <- '~/geolab_storage_V3/data/Analogs/ECMWF_Italy/ItalyAnalysis_new.nc'
nc.analysis <- nc_open(nc.analy.file)
obsv <- ncvar_get(nc.analysis, 'Data')
obsv.times <- ncvar_get(nc.analysis, "Times")
# Calculate WS and WD from u and v components
obsv.calc <- obsv
u.10.obsv <- obsv[1,,]
v.10.obsv <- obsv[2,,]
dir.10.obsv <- UVtoDir(u.10.obsv, v.10.obsv)
spd.10.obsv <- UVtoSpd(u.10.obsv, v.10.obsv)
obsv.calc[1,,] <- dir.10.obsv
obsv.calc[2,,] <- spd.10.obsv
# ANEN PARAMETERS
# Define variables here:
members.size <- 21
# Choose variable to be predicted (predictand)
predictandParam <- 3 # parameter 1 is WD; param 2 is WS; param 3 is Temperature
stations.ID <- 1:2499
weights <- rep(1, dim(fcst.aligned)[1]) # <- c(1,1,0,0,0) # "wdir","ws","2T","2DPT","MSLP"
verbosity <- 5
preserve_mapping <- TRUE
extObs <- FALSE
# operational <- TRUE # By default this is FALSE.
test.start <- 731
test.end <- 1095
search.start <- 1
search.end <- 730
rm(nc.analysis,nc.fcst,u.10.obsv,v.10.obsv,dir.10.fcst,dir.10.obsv,nc.analy.file,nc.fcst.file,spd.10.fcst,spd.10.obsv)
xs <- as.numeric(lon)
ys <- as.numeric(lat)
nx <- 51; ny <- 49 # Preset for Italy dataset
config <- generateConfiguration('extendedSearch')
config$observation_id <- predictandParam
config$test_forecasts <- fcst.aligned
config$test_times <- fcst.times # NEW
config$search_forecasts <- fcst.aligned
config$search_times <- fcst.times
config$flts <- flts.subset
config$search_observations <- obsv
config$observation_times <- obsv.times
config$num_members <- members.size
config$weights <- weights
config$test_stations_x <- xs[stations.ID]
config$test_stations_y <- ys[stations.ID]
config$search_stations_x <- xs[stations.ID]
config$search_stations_y <- ys[stations.ID]
config$preserve_mapping <- preserve_mapping
config$verbose <- verbosity
config$extend_observations <- extObs
# Set up test times to be compared
config$test_times_compare <- config$search_times[test.start:test.end]
config$search_times_compare <- config$search_times[search.start:search.end] # This means nothing if operational is changed to TRUE. However, operational is FALSE by default.
# Could also call fcst.times instead but then it says "ERROR: Test times for comparison should be a numeric vector!"# [1] "Please use is.vector() and is.numeric() to check!
# And does not validate
# Specific to SSE
config$preserve_search_stations <- T
config$preserve_similarity <- T
config$num_nearest <- 8
config$max_num_search_stations <- 10
# Validate first before using
validateConfiguration(config)
# w/ search extension
AnEn <- generateAnalogs(config)
Weiming-Hu
Following use of Parallel Ensemble help page (https://weiming-hu.github.io/AnalogsEnsemble/2019/02/12/operational-search.html) and binder (https://hub.mybinder.org/user/weiming-hu-analogsensemble-bmqvhvn1/notebooks/demo-3_operational-search.ipynb) documentation, 1) request help utilizing new commands added/revised with the inclusion of the Operational Search option 2) Suggest revision of user interface to reduce duplication and streamline use