Automate "ncvar_get"-reading of different variable names in NetCDF files - r

Question

I have a NetCDF dataset with two climate scenarios (rcp & hist), both of them containing 25 files. Each file either contains data for the variable "pr", "tas", "tasmax", or "tasmin". I wrote a for loop to iteratively read the files of hist and rcp, read them with nc_open, extract the variable with ncvar_get and finally make a calculation in form of mean(abs(hist - rcp) to obtain the mean absolute distance between each pair of hist and rcp. The problem: as ncvar_get requires the exact variable name of the current file I wrote an if else block (see below) that shall find the variable name of the current file and apply it for ncvar_get. Running the code I obtain the following error:
[1] "vobjtovarid4: error #F: I could not find the requsted var (or dimvar) in the file!"
[1] "var (or dimvar) name: tas"
[1] "file name: /data/historical/tasmax_ICHEC-EC-EARTH_DMI-HIRHAM5_r3i1p1.nc" Error in vobjtovarid4(nc, varid, verbose = verbose, allowdimvar = TRUE) : Variable not found
#Extract of the files in the hist list. Same file names in the rcp list, but different directory
> hist.files.cl <- list.files("/historical", full.names = TRUE)
> hist.files.cl
[1] "/historical/pr_CNRM-CERFACS-CNRM-CM5_ALADIN53_r1i1p1.nc"
[2] "/historical/pr_CNRM-CERFACS-CNRM-CM5_ALARO-0_r1i1p1.nc"
[3] "/historical/pr_ICHEC-EC-EARTH_HIRHAM5_r3i1p1.nc"
[4] "/historical/pr_ICHEC-EC-EARTH_RACMO22E_r12i1p1.nc"
[5] "/historical/pr_ICHEC-EC-EARTH_RCA4_r12i1p1.nc"
[6] "/historical/pr_MPI-M-MPI-ESM-LR_RCA4_r1i1p1.nc"
[7] "/historical/pr_MPI-M-MPI-ESM-LR_REMO2009_r1i1p1.nc"
[8] "/historical/pr_MPI-M-MPI-ESM-LR_REMO2009_r2i1p1.nc"
[9] "/historical/tas_CNRM-CERFACS-CNRM-CM5_CNRM-ALADIN53_r1i1p1.nc"
[10] "/historical/tas_CNRM-CERFACS-CNRM-CM5_RMIB-UGent-ALARO-0_r1i1p1.nc"
[11] "/historical/tas_ICHEC-EC-EARTH_DMI-HIRHAM5_r3i1p1.nc"
[12] "/historical/tas_ICHEC-EC-EARTH_KNMI-RACMO22E_r12i1p1.nc"
[13] "/historical/tas_ICHEC-EC-EARTH_SMHI-RCA4_r12i1p1.nc"
[14] "/historical/tas_MPI-M-MPI-ESM-LR_MPI-CSC-REMO2009_r1i1p1.nc"
[15] "/historical/tas_MPI-M-MPI-ESM-LR_MPI-CSC-REMO2009_r2i1p1.nc"
[16] "/historical/tasmax_ICHEC-EC-EARTH_DMI-HIRHAM5_r3i1p1.nc"
[17] "/historical/tasmax_ICHEC-EC-EARTH_KNMI-RACMO22E_r12i1p1.nc"
[18] "/historical/tasmax_ICHEC-EC-EARTH_SMHI-RCA4_r12i1p1.nc"
euc.distance <- list()
for(i in 1:length(hist.files.cl)) {
#Open ith file in list of hist files as well as in list of rcp files
hist.data <- nc_open(hist.files.cl[i])
rcp.data <- nc_open(rcp.files.cl[i])
if(grepl("pr", hist.data$filename)){
hist.var <- ncvar_get(hist.data, "pr")
rcp.var <- ncvar_get(rcp.data, "pr")
}else if (grepl("tas", hist.data$filename)){
hist.var <- ncvar_get(hist.data, "tas")
rcp.var <- ncvar_get(rcp.data, "tas")
}else if (grepl("tasmax", hist.data$filename)){
hist.var <- ncvar_get(hist.data, "tasmax")
rcp.var <- ncvar_get(rcp.data, "tasmax")
}else{
hist.var <- ncvar_get(hist.data, "tasmin")
rcp.var <- ncvar_get(rcp.data, "tasmin")
}
#Converting temperature variable from K to °C:
if(grepl("tas", hist.data$filename)){
hist.var <- hist.var-273.15
rcp.var <- rcp.var-273.15
}
#Find for the ith rcp file with dim=(1,1,360) in the ith hist file with dim=(385,373,360) the grid point with the best fitting distribution (each grid point consists of a distribution of 360 time steps).The calculation may contain errors...
euc.distance[[i]] <- apply(hist.var, c(1,2), function(x) mean(abs(rcp.var - x)))
min_values <- which(rank(euc.distance[[i]], ties.method='min') <= 10)
}
As cath highlighted the probable cause of the error, but the proposed approach to extract the part of interest (=variable name) from the filename does not work. I before tried to automate the extraction of the variable name by using stringr("filename",startposition, endposition) until I noticed that there is no sense in it, because each variable name (pr, tas, tasmax, tasmin) has another string length. What possibilities do you see for me?
Thank you a lot!

Answer 1

To complete a bit my comment, if you need to operate on each file, you could do it at once, putting everything in a list.
So, first get the "keypart" for each file:
keyparts <- sub("^([a-z]+)_.+", "\\1", basename(hist.files.cl))
keyparts
# [1] "pr" "pr" "pr" "pr" "pr" "pr" "pr" "pr"
# [9] "tas" "tas" "tas" "tas" "tas" "tas" "tas" "tasmax"
#[17] "tasmax" "tasmax"
Then you can use lapply to do what you need to do for every files at once:
my_res <- lapply(seq(keyparts),
function(i){
hist.data <- nc_open(hist.files.cl[i])
rcp.data <- nc_open(rcp.files.cl[i])
hist.var <- ncvar_get(hist.data, keyparts[i])
rcp.var <- ncvar_get(rcp.data, keyparts[i])
if(keyparts[i]=="tas"){
hist.var <- hist.var-273.15
rcp.var <- rcp.var-273.15
}
euc.distance <- apply(hist.var, c(1,2), function(x) mean(abs(rcp.var - x)))
min_values <- which(rank(euc.distance[[i]], ties.method='min') <= 10)
return(list(euc.distance=euc.distance, min.values=min.values))
})