I am trying to loop the
ncin_old<-nc_open("filename", write=TRUE, readunlim=TRUE, verbose=FALSE,
auto_GMT=TRUE, suppress_dimvals=FALSE )
function like this
library(ncdf.tools)
library(ncdf4)
library(ncdf4.helpers)
library(RNetCDF)
library(abind)
setwd("D:/Rwork/Project") # set working folder
# This is the directory where the file for analysing are
dir("D:/Rwork/Project/MASTER_FILES")-> xlab
filelist <- paste("MASTER_FILES/", dir("MASTER_FILES"), sep="")
N <- length(filelist) # Loop over the individual files
for(j in 1:N) {
ncin_old <- nc_open("filelist[j]", write=TRUE, readunlim=TRUE, verbose=FALSE,
auto_GMT=TRUE, suppress_dimvals=FALSE )
}
But I get this error
Error in nc_open("filelist[j]", write = TRUE, readunlim = TRUE,
verbose = FALSE, : Error in nc_open trying to open file
filelist[j]
If I drop everything after filelist[j] the lat file in the loop opens
but the nc_open(x, write) does not seem to like being looped.
I have fixed some issues of your code as below. I think now it is correct.
library(ncdf4)
# set the folder with the files
setwd("D:/Rwork/Project/MASTER_FILES")
# you need the files path, not the directory path
# list only the files with the .nc extension
filelist <- list.files(pattern = "\\.nc$")
# Loop over the individual files
# The filelist cannot be between quotation marks as in your code
N <- length(filelist)
for(j in 1:N) {
ncin_old <- nc_open(filelist[j], write=TRUE, readunlim=TRUE, verbose=FALSE,
auto_GMT=TRUE, suppress_dimvals=FALSE)
}
I've used lapply:
library(ncdf4)
# set the folder that contains all the files
setwd("C:/...")
# create a list with the files with the .nc extension
filelist <- list.files(pattern = "*.nc")
filelist # It contains all the files .nc
# To open all files: Loop over the individual files
for (i in 1:length(filelist)) {
all_nc_files <- lapply(filelist, nc_open)
}
Running that, I obtain "all_nc_files", which contains all .nc files opened and now I can work with them.
Hope it works!
Related
I am so close to getting my code to work, but cannot seem to figure out how to get a dynamic file name. Here is what Ivve got:
require(ncdf)
require(raster)
require(rgdal)
## For multiple files, use a for loop
## Input directory
dir.nc <- 'inputdirectoy'
files.nc <- list.files(dir.nc, full.names = T, recursive = T)
## Output directory
dir.output <- 'outputdirectory'
## For simplicity, I use "i" as the file name, but would like to have a dynamic one
for (i in 1:length(files.nc)) {
r.nc <- raster(files.nc[i], varname = "precipitation")
writeRaster(r.nc, paste(dir.output, i, '.tiff', sep = ''), format = 'GTiff', prj = T, overwrite = T)
}
## END
I appreciate any help. So close!!
You can do this in different ways, but I think it is generally easiest to first create all the output filenames (and check if they are correct) and then use these in the loop.
So something like this:
library(raster)
infiles <- list.files('inputpath', full.names=TRUE)
ff <- extension(basename(infiles), '.tif')
outpath <- 'outputpath'
outfiles <- file.path(outpath, ff)
To assure that you are writing to an existing folder, you can create it first.
dir.create(outpath, showWarnings=FALSE, recursive=TRUE)
And then loop over the files
for (i in 1:length(infiles)) {
r <- raster(infiles[i])
writeRaster(r, paste(outfiles[i], overwrite = TRUE)
}
You might also use something along these lines
outfiles <- gsub('in', 'out', infiles)
Here is the code that finally worked:
# Imports
library(raster)
#Set source file
infiles <- list.files('infilepath', full.names=TRUE)
#create dynamic file names and choose outfiles to view list
ff <- extension(basename(infiles), '.tif')
outpath <- 'outfilepath'
outfiles <- file.path(outpath, ff)
#run da loop
for (i in 1:length(infiles)) {
r <- raster(infiles[i])
writeRaster(r, paste(outfiles[i]), format ='GTiff', overwrite = T)
}
## END
I have many .csv files in a folder. I want to get the binning result from each of the .csv file one by one automatically by R scripting from command line, and one by one write the result of all files into result.csv file. For example, I have file01.csv, file02.csv, file03.csv, file04.csv, file05.csv. I want that first R script will read / execute file01.csv and write the result into result.csv file, then read / execute file02.csv and write result into result.csv, again read / execute file03.csv and write result into result.csv, and so on. This is like a loop on all the files, and I want to execute the R script from the command line.
Here is my starting R script:
data <- read.table("file01.csv",sep=",",header = T)
df.train <- data.frame(data)
library(smbinning) # Install if necessary
<p>#Analysis by dwell:</p>
df.train_amp <-
rbind(df.train)
res.bin <- smbinning(df=df.train_amp, y="cvflg",x="dwell")
res.bin #Result
<p># Analysis by pv</p>
df.train_amp <-
rbind(df.train)
res.bin <- smbinning(df=df.train_amp, y="cvflg",x="pv")
res.bin #Result
Any suggestion and support would be appreciated highly.
Thank
Firstly you will want to read in the files from your directory. Place all of your source files in the same source directory. I am assuming here that your CSV files all have the same shape. Also, I am doing nothing about headers here.
directory <- "C://temp" ## for example
filenames <- list.files(directory, pattern = "*.csv", full.names = TRUE)
# If you need full paths then change the above to
# filenames <- list.files(directory, pattern = "*.csv", full.names = TRUE)
bigDF <- data.frame()
for (f in 1:length(filenames)){
tmp <- read.csv(paste(filenames[f]), stringsAsFactors = FALSE)
bigDF <- rbind(bigDF, tmp)
}
This will add the rows in tmp to bigDF for each read, and should result in final bigDF.
To write the df to a csv is trivial in R as well. Anything like
# Write to a file, suppress row names
write.csv(bigDF, "myData.csv", row.names=FALSE)
# Same, except that instead of "NA", output blank cells
write.csv(bigDF, "myData.csv", row.names=FALSE, na="")
# Use tabs, suppress row names and column names
write.table(bigDF, "myData.csv", sep="\t", row.names=FALSE, col.names=FALSE)
Finally I find the above problem can be solved as follows:
library(smbinning) #Install if necessary。
files <- list.files(pattern = ".csv") ## creates a vector with all files names in your folder
cutpoint <- rep(0,length(files))
for(i in 1:length(files)){
data <- read.csv(files[i],header=T)
df.train <- data.frame(data)
df.train_amp <- rbind(df.train,df.train,df.train,df.train,df.train,df.train,df.train,df.train) # Just to multiply the data
cutpoint[i] <- smbinning(df=df.train_amp, y="cvflg",x="dwell") # smbinning is calculated here
}
result <- cbind(files,cutpoint) # Produce details results
result <- cbind(files,bands) # Produce bands results
write.csv(result,"result_dwell.csv") # write result into csv file
I'm sure this must have been answered somewhere, so; if you have a pointer to an answer that helps please let me know... ;o)
I have a number of fairly sizeable processing tasks (mainly multi-label text classifiers) which read in large volumes of files, do stuff with that, output a result then move on to the next.
I have this working neatly sequentially but wanted to parallelise things.
By way of a really basic example...
require(plyr)
fileDir <- "/Users/barneyc/sourceFiles"
outputDir <- "/Users/barneyc/outputFiles"
files <- as.list(list.files(full.names=TRUE,recursive=FALSE,pattern=".csv"))
l_ply(files, function(x){
print(x)
#change to dir containing source files
setwd(fileDir)
# read file
content <- read.csv(file=x,header=TRUE)
# change directory to output
setwd(outputDir)
# append the itemID from CSV file to
write.table(content$itemID,file="ids.csv", append = TRUE, sep=",", row.names=FALSE,col.names=TRUE)
}, .parallel=FALSE )
Will iterate through all the files in directory fileDir, opening each CSV, extracting a value from the file and appending this to an output CSV held in the directory outputDir. A basic example but it runs just fine to illustrate the problem.
To run this in parallel creates a problem in so far as the directory variables (fileDir & outputDir) are essentially unknown by the anonymous function (x), ala...
require(plyr)
require(doParallel)
fileDir <- "/Users/barneyc/sourceFiles"
outputDir <- "/Users/barneyc/outputFiles"
files <- as.list(list.files(full.names=TRUE,recursive=FALSE,pattern=".csv"))
cl<-makeCluster(4) # make a cluster of available cores
registerDoParallel(cl) # raise cluster
l_ply(files, function(x){
print(x)
#change to dir containing source files
#setwd(fileDir)
# read file
content <- read.csv(file=x,header=TRUE)
# change directory to output
setwd(y)
# append the itemID from CSV file to
write.table(content$itemID,file="ids.csv", append = TRUE, sep=",", row.names=FALSE,col.names=TRUE)
}, .parallel=TRUE )
stopCluster() # kill the cluster
Can anyone shed light on how I pass those two directory variables through to the function here?
So thanks to #Roland my parallel function would now be...
require(plyr)
require(doParallel)
fileDir <- "/Users/barneyc/sourceFiles"
outputDir <- "/Users/barneyc/outputFiles"
files <- as.list(list.files(full.names=TRUE,recursive=FALSE,pattern=".csv"))
cl<-makeCluster(4) # make a cluster of available cores
registerDoParallel(cl) # raise cluster
l_ply(files, function(x,y,z){
filename <- x
fileDir <- y
outputDir <- z
#change to dir containing source files
setwd(fileDir)
# read file
content <- read.csv(file=filename,header=TRUE)
# change directory to output
setwd(outputDir)
# append the itemID from CSV file to
write.table(content$itemID,file="ids.csv", append = TRUE, sep=",", row.names=FALSE,col.names=TRUE)
}, y=fileDir, z=outputDir, .parallel=TRUE )
stopCluster() # kill the cluster
I have used a for loop to load some files, perform some tasks and write the results to a new csv file. The directory where the input files are stored was set before running the loop, but rather than saving the output csv file to the same directory, I would like to send it to a new directory within the loop.
Here is a very simple for loop example:
p <- "~/Desktop/MyFolder"
setwd(p)
files <- list.files(path=dir, pattern="csv$", full.names=FALSE, recursive=FALSE)
for(i in 1:length(files)){
f <- lapply(files[i], read.csv, header=TRUE, stringsAsFactors=FALSE)
cat2 <- f[f$mod ==2, c(1,6)]
filename <- files[1]
tn <- strsplit(filename,"_")[[1]][1]
fn <- paste(tn, "_trimmed.csv", sep="")
write.csv(cat2, file=fn, row.names=FALSE)
}
I am quite new to R and have not been able to find out how to do this. Any help would be appreciated.
So assuming that the folders don't already exist, and you want to create them and then put output.csv in each, the following should work:
p <- "~/Desktop/MyFolder"
setwd(p)
files <- list.files(path=dir, pattern="csv$", full.names=FALSE, recursive=FALSE)
for(i in 1:length(files)){
f <- lapply(files[i], read.csv, header=TRUE, stringsAsFactors=FALSE)
#why not:
#f <- read.csv(files[i],header=TRUE, stringsAsFactors=FALSE)
cat2 <- f[f$mod ==2, c(1,6)]
dir.create(paste0("folder",i), showWarnings = FALSE) #stops warnings if folder already exists
write.csv(cat2, file.path(paste0("folder",i), "output.csv"), row.names=FALSE)
}
Your lapply() statement looks unnecessary to me, but maybe there's something that I'm missing
I need to automate R to read a csv datafile that's into a zip file.
For example, I would type:
read.zip(file = "myfile.zip")
And internally, what would be done is:
Unzip myfile.zip to a temporary folder
Read the only file contained on it using read.csv
If there is more than one file into the zip file, an error is thrown.
My problem is to get the name of the file contained into the zip file, in orded to provide it do the read.csv command. Does anyone know how to do it?
UPDATE
Here's the function I wrote based on #Paul answer:
read.zip <- function(zipfile, row.names=NULL, dec=".") {
# Create a name for the dir where we'll unzip
zipdir <- tempfile()
# Create the dir using that name
dir.create(zipdir)
# Unzip the file into the dir
unzip(zipfile, exdir=zipdir)
# Get the files into the dir
files <- list.files(zipdir)
# Throw an error if there's more than one
if(length(files)>1) stop("More than one data file inside zip")
# Get the full name of the file
file <- paste(zipdir, files[1], sep="/")
# Read the file
read.csv(file, row.names, dec)
}
Since I'll be working with more files inside the tempdir(), I created a new dir inside it, so I don't get confused with the files. I hope it may be useful!
Another solution using unz:
read.zip <- function(file, ...) {
zipFileInfo <- unzip(file, list=TRUE)
if(nrow(zipFileInfo) > 1)
stop("More than one data file inside zip")
else
read.csv(unz(file, as.character(zipFileInfo$Name)), ...)
}
You can use unzip to unzip the file. I just mention this as it is not clear from your question whether you knew that. In regard to reading the file. Once your extracted the file to a temporary dir (?tempdir), just use list.files to find the files that where dumped into the temporary directory. In your case this is just one file, the file you need. Reading it using read.csv is then quite straightforward:
l = list.files(temp_path)
read.csv(l[1])
assuming your tempdir location is stored in temp_path.
I found this thread as I was trying to automate reading multiple csv files from a zip. I adapted the solution to the broader case. I haven't tested it for weird filenames or the like, but this is what worked for me so I thought I'd share:
read.csv.zip <- function(zipfile, ...) {
# Create a name for the dir where we'll unzip
zipdir <- tempfile()
# Create the dir using that name
dir.create(zipdir)
# Unzip the file into the dir
unzip(zipfile, exdir=zipdir)
# Get a list of csv files in the dir
files <- list.files(zipdir)
files <- files[grep("\\.csv$", files)]
# Create a list of the imported csv files
csv.data <- sapply(files, function(f) {
fp <- file.path(zipdir, f)
return(read.csv(fp, ...))
})
return(csv.data)}
If you have zcat installed on your system (which is the case for linux, macos, and cygwin) you could also use:
zipfile<-"test.zip"
myData <- read.delim(pipe(paste("zcat", zipfile)))
This solution also has the advantage that no temporary files are created.
Here is an approach I am using that is based heavily on #Corned Beef Hash Map 's answer. Here are some of the changes I made:
My approach makes use of the data.table package's fread(), which
can be fast (generally, if it's zipped, sizes might be large, so you
stand to gain a lot of speed here!).
I also adjusted the output format so that it is a named list, where
each element of the list is named after the file. For me, this was a
very useful addition.
Instead of using regular expressions to sift through the files
grabbed by list.files, I make use of list.file()'s pattern
argument.
Finally, I by relying on fread() and by making pattern an
argument to which you could supply something like "" or NULL or
".", you can use this to read in many types of data files; in fact,
you can read in multiple types of at once (if your .zip contains
.csv, .txt in you want both, e.g.). If there are only some types of
files you want, you can specify the pattern to only use those, too.
Here is the actual function:
read.csv.zip <- function(zipfile, pattern="\\.csv$", ...){
# Create a name for the dir where we'll unzip
zipdir <- tempfile()
# Create the dir using that name
dir.create(zipdir)
# Unzip the file into the dir
unzip(zipfile, exdir=zipdir)
# Get a list of csv files in the dir
files <- list.files(zipdir, rec=TRUE, pattern=pattern)
# Create a list of the imported csv files
csv.data <- sapply(files,
function(f){
fp <- file.path(zipdir, f)
dat <- fread(fp, ...)
return(dat)
}
)
# Use csv names to name list elements
names(csv.data) <- basename(files)
# Return data
return(csv.data)
}
The following refines the above answers. FUN could be read.csv, cat, or anything you like, providing the first argument will accept a file path. E.g.
head(read.zip.url("http://www.cms.gov/Medicare/Coding/ICD9ProviderDiagnosticCodes/Downloads/ICD-9-CM-v32-master-descriptions.zip", filename = "CMS32_DESC_LONG_DX.txt"))
read.zip.url <- function(url, filename = NULL, FUN = readLines, ...) {
zipfile <- tempfile()
download.file(url = url, destfile = zipfile, quiet = TRUE)
zipdir <- tempfile()
dir.create(zipdir)
unzip(zipfile, exdir = zipdir) # files="" so extract all
files <- list.files(zipdir)
if (is.null(filename)) {
if (length(files) == 1) {
filename <- files
} else {
stop("multiple files in zip, but no filename specified: ", paste(files, collapse = ", "))
}
} else { # filename specified
stopifnot(length(filename) ==1)
stopifnot(filename %in% files)
}
file <- paste(zipdir, files[1], sep="/")
do.call(FUN, args = c(list(file.path(zipdir, filename)), list(...)))
}
Another approach that uses fread from the data.table package
fread.zip <- function(zipfile, ...) {
# Function reads data from a zipped csv file
# Uses fread from the data.table package
## Create the temporary directory or flush CSVs if it exists already
if (!file.exists(tempdir())) {dir.create(tempdir())
} else {file.remove(list.files(tempdir(), full = T, pattern = "*.csv"))
}
## Unzip the file into the dir
unzip(zipfile, exdir=tempdir())
## Get path to file
file <- list.files(tempdir(), pattern = "*.csv", full.names = T)
## Throw an error if there's more than one
if(length(file)>1) stop("More than one data file inside zip")
## Read the file
fread(file,
na.strings = c(""), # read empty strings as NA
...
)
}
Based on the answer/update by #joão-daniel
unzipped file location
outDir<-"~/Documents/unzipFolder"
get all the zip files
zipF <- list.files(path = "~/Documents/", pattern = "*.zip", full.names = TRUE)
unzip all your files
purrr::map(.x = zipF, .f = unzip, exdir = outDir)
I just wrote a function based on top read.zip that may help...
read.zip <- function(zipfile, internalfile=NA, read.function=read.delim, verbose=TRUE, ...) {
# function based on http://stackoverflow.com/questions/8986818/automate-zip-file-reading-in-r
# check the files within zip
unzfiles <- unzip(zipfile, list=TRUE)
if (is.na(internalfile) || is.numeric(internalfile)) {
internalfile <- unzfiles$Name[ifelse(is.na(internalfile),1,internalfile[1])]
}
# Create a name for the dir where we'll unzip
zipdir <- tempfile()
# Create the dir using that name
if (verbose) catf("Directory created:",zipdir,"\n")
dir.create(zipdir)
# Unzip the file into the dir
if (verbose) catf("Unzipping file:",internalfile,"...")
unzip(zipfile, file=internalfile, exdir=zipdir)
if (verbose) catf("Done!\n")
# Get the full name of the file
file <- paste(zipdir, internalfile, sep="/")
if (verbose)
on.exit({
catf("Done!\nRemoving temporal files:",file,".\n")
file.remove(file)
file.remove(zipdir)
})
else
on.exit({file.remove(file); file.remove(zipdir);})
# Read the file
if (verbose) catf("Reading File...")
read.function(file, ...)
}