Creating a function to loop columns through an equation in R - r
Solution (thanks #Peter_Evan!) in case anyone coming across this question has a similar issue
(Original question is below)
## get all slopes (lm coefficients) first
# list of subfields of interest to loop through
sf <- c("left_presubiculum", "right_presubiculum",
"left_subiculum", "right_subiculum", "left_CA1", "right_CA1",
"left_CA3", "right_CA3", "left_CA4", "right_CA4", "left_GC-ML-DG",
"right_GC-ML-DG")
# dependent variables are sf, independent variable common to all models in the inner lm() call is ICV
# applies the lm(subfield ~ ICV, dataset = DF) to all subfields of interest (sf) specified previously
lm.results <- lapply(sf, function(dv) {
temp.lm <- lm(get(dv) ~ ICV, data = DF)
coef(temp.lm)
})
# returns a list, where each element is a vector of coefficients
# do.call(rbind, ) will paste them together
lm.coef <- data.frame(sf = sf,
do.call(rbind, lm.results))
# tidy up name of intercept variable
names(lm.coef)[2] <- "intercept"
lm.coef
## set up all components for the equation
# matrix to store output
out <- matrix(ncol = length(sf), nrow = NROW(DF))
# name the rows after each subject
row.names(out) <- DF$Subject
# name the columns after each subfield
colnames(out) <- sf
# nested for loop that goes by subject (j) and subfield (i)
for(j in DF$Subject){
for (i in sf) {
slope <- lm.coef[lm.coef$sf == i, "ICV"]
out[j,i] <- as.numeric( DF[DF$Subject == j, i] - (slope * (DF[DF$Subject == j, "ICV"] - mean(DF$ICV))) )
}
}
# check output
out
===============
Original Question:
I have a dataframe (DF) with 13 columns (12 different brain subfields, and one column containing total intracranial volume(ICV)) and 50 rows (each a different participant). I'm trying to automate an equation being looped over every column for each participant.
The data:
structure(list(Subject = c("sub01", "sub02", "sub03", "sub04",
"sub05", "sub06", "sub07", "sub08", "sub09", "sub10", "sub11",
"sub12", "sub13", "sub14", "sub15", "sub16", "sub17", "sub18",
"sub19", "sub20"), ICV = c(1.50813, 1.3964237, 1.6703585, 1.4641886,
1.6351018, 1.5524641, 1.4445532, 1.6384505, 1.6152434, 1.5278011,
1.4788126, 1.4373356, 1.4109637, 1.3634952, 1.3853583, 1.4855268,
1.6082085, 1.5644998, 1.5617522, 1.4304141), left_subiculum = c(411.225013,
456.168033, 492.968477, 466.030173, 533.95505, 476.465524, 448.278213,
476.45566, 422.617374, 498.995121, 450.773906, 461.989663, 549.805272,
452.619547, 457.545623, 451.988333, 475.885847, 490.127968, 470.686415,
494.06548), left_CA1 = c(666.893596, 700.982955, 646.21927, 580.864234,
721.170599, 737.413139, 737.683665, 597.392434, 594.343911, 712.781376,
733.157168, 699.820162, 701.640861, 690.942843, 606.259484, 731.198846,
567.70879, 648.887718, 726.219904, 712.367433), left_presubiculum = c(325.779458,
391.252815, 352.765098, 342.67797, 390.885737, 312.857458, 326.916867,
350.657957, 325.152464, 320.718835, 273.406949, 305.623938, 371.079722,
315.058313, 311.376271, 319.56678, 348.343569, 349.102678, 322.39908,
306.966008), `left_GC-ML-DG` = c(327.037756, 305.63224, 328.945065,
238.920358, 319.494513, 305.153183, 311.347404, 259.259723, 295.369164,
312.022281, 324.200989, 314.636501, 306.550385, 311.399107, 295.108592,
356.197094, 251.098248, 294.76349, 317.308576, 301.800253), left_CA3 = c(275.17038,
220.862237, 232.542718, 170.088695, 234.707172, 210.803287, 246.861975,
171.90896, 220.83478, 236.600832, 246.842024, 239.677362, 186.599097,
224.362411, 229.9142, 293.684776, 172.179779, 202.18936, 232.5666,
221.896625), left_CA4 = c(277.614028, 264.575987, 286.605092,
206.378619, 281.781858, 258.517989, 269.354864, 226.269982, 256.384436,
271.393257, 277.928824, 265.051581, 262.307377, 266.924683, 263.038686,
306.133918, 226.364556, 262.42823, 264.862956, 255.673948), right_subiculum = c(468.762375,
445.35738, 446.536018, 456.73484, 521.041823, 482.768261, 487.2911,
456.39996, 445.392976, 476.146498, 451.775611, 432.740085, 518.170065,
487.642399, 405.564237, 487.188989, 467.854363, 479.268714, 473.212833,
472.325916), right_CA1 = c(712.973011, 717.815214, 663.637105,
649.614586, 711.844375, 779.212704, 862.784416, 648.925038, 648.180611,
760.761704, 805.943016, 717.486756, 801.853608, 722.213109, 621.676321,
791.672796, 605.35667, 637.981476, 719.805053, 722.348921), right_presubiculum = c(327.285242,
364.937865, 288.322641, 348.30058, 341.309111, 279.429847, 333.096795,
342.184296, 364.245998, 350.707173, 280.389853, 276.423658, 339.439377,
321.534798, 302.164685, 328.365751, 341.660085, 305.366589, 320.04127,
303.83284), `right_GC-ML-DG` = c(362.391907, 316.853532, 342.93274,
282.550769, 339.792696, 357.867386, 342.512721, 277.797528, 309.585721,
343.770416, 333.524912, 302.505077, 309.063135, 291.29361, 302.510461,
378.682679, 255.061044, 302.545288, 313.93902, 297.167161), right_CA3 = c(307.007404,
243.839349, 269.063801, 211.336979, 249.283479, 276.092623, 268.183349,
202.947849, 214.642782, 247.844657, 291.206598, 235.864996, 222.285729,
201.427853, 237.654913, 321.338801, 199.035108, 243.204203, 236.305659,
213.386702), right_CA4 = c(312.164065, 272.905586, 297.99392,
240.765062, 289.98697, 306.459566, 284.533068, 245.965817, 264.750571,
296.149675, 290.66935, 264.821461, 264.920869, 246.267976, 266.07378,
314.205819, 229.738951, 274.152503, 256.414608, 249.162404)), row.names = c(NA,
-20L), class = c("tbl_df", "tbl", "data.frame"))
The equation:
adjustedBrain(participant1) = rawBrain(participant1) - slope*[ICV(participant1) - (mean of all ICV measures included in the calculation of the slope)]
The code (which is not working and I was hoping for some pointers):
adjusted_Brain <- function(DF, subject) {
subfields <- colnames(select(DF, "left_presubiculum", "right_presubiculum",
"left_subiculum", "right_subiculum", "left_CA1", "right_CA1",
"left_CA3", "right_CA3", "left_CA4", "right_CA4", "left_GC-ML-DG",
"right_GC-ML-DG"))
out <- matrix(ncol = length(subfields), nrow = NROW(DF))
for (i in seq_along(subfields)) {
DF[i] = DF[DF$Subject == "subject", "i"] -
slope * (DF[DF$Subject == "subject", "ICV"] -
mean(DF$ICV))
}
}
Getting this error:
Error: Can't subset columns that don't exist.
x Column `i` doesn't exist.
A few notes:
The slopes for each subject for each subfield will be different (and will come from a regression) -> is there a way to specify that in the function so the slope (coefficient from the appropriate regression equation) gets called in?
I have my nrow set to the number of participants right now in the output because I'd like to have this run through EVERY subject across EVERY subfield and spit out a matrix with all the adjusted brain volumes... But that seems very complicated and so for now I will just settle for running each participant separately.
Any help is greatly appreciated!
As others have noted in the comments, there are quite a few syntax issues that prevent your code from running, as well as a few unstated requirements. That aside, I think there is enough to recommend a few improvements that you can hopefully build on. Here are the top line changes:
You likely don't need this to be a function, but rather a nested for loop (if you want to do this with base R). As written, the code isn't flexible enough to merit a function. If you intend to apply this many times across different datasets, a function might make sense. However, it will require a much larger rewrite.
Assuming you are fitting a simple regression via lm, then you can pull out the coefficient of interest via the $ operator and indexing (see below). Some thought will need to go into how to handle different models in the loop. Here, we assume you only need one coefficient from one model.
There are a few areas where the syntax is incorrect and a review of sub setting in base R would be helpful. Others have pointed out in the comments were some of these are.
Here is one approach were we loop through each subject (j) through each feature or subfield (i) and store them in a matrix (out). This is just an approach and will almost certainly need tweaking on your end!
#NOTE: the dataset your provided is saved as x in this example.
#fit a linear model - here we assume there is only one coef. of interest, but you may need to alter
# depending on how the slope changes in each calculation
reg <- lm(ICV ~ right_CA3, x)
# view the coeff.
reg$coefficients
# pull out the slope by getting the coeff. of interest (via index) from the reg object
slope <- reg$coefficients[[1]]
# list of features/subfeilds to loop through
sf <- c("left_presubiculum", "right_presubiculum",
"left_subiculum", "right_subiculum", "left_CA1", "right_CA1",
"left_CA3", "right_CA3", "left_CA4", "right_CA4", "left_GC-ML-DG",
"right_GC-ML-DG")
# matrix to store output
out <- matrix(ncol = length(sf), nrow = NROW(x))
#name the rows after each subject
row.names(out) <- x$Subject
#name the columns after each sub feild
colnames(out) <- sf
# nested for loop that goes by subject (j) and features/subfeilds (i)
for(j in x$Subject){
for (i in sf) {
out[j,i] <- as.numeric( x[x$Subject == j, i] - (slope * (x[x$Subject == j, "ICV"] - mean(x$ICV))) )
}
}
# check output
out
Related
Using cpquery function for several pairs from dataset
I am relatively beginner in R and trying to figure out how to use cpquery function for bnlearn package for all edges of DAG. First of all, I created a bn object, a network of bn and a table with all strengths. library(bnlearn) data(learning.test) baynet = hc(learning.test) fit = bn.fit(baynet, learning.test) sttbl = arc.strength(x = baynet, data = learning.test) Then I tried to create a new variable in sttbl dataset, which was the result of cpquery function. sttbl = sttbl %>% mutate(prob = NA) %>% arrange(strength) sttbl[1,4] = cpquery(fit, `A` == 1, `D` == 1) It looks pretty good (especially on bigger data), but when I am trying to automate this process somehow, I am struggling with errors, such as: Error in sampling(fitted = fitted, event = event, evidence = evidence, : logical vector for evidence is of length 1 instead of 10000. In perfect situation, I need to create a function that fills the prob generated variable of sttbl dataset regardless it's size. I tried to do it with for loop to, but stumbled over the error above again and again. Unfortunately, I am deleting failed attempts, but they were smt like this: for (i in 1:nrow(sttbl)) { j = sttbl[i,1] k = sttbl[i,2] sttbl[i,4]=cpquery(fit, fit$j %in% sttbl[i,1]==1, fit$k %in% sttbl[i,2]==1) } or this: for (i in 1:nrow(sttbl)) { sttbl[i,4]=cpquery(fit, sttbl[i,1] == 1, sttbl[i,2] == 1) } Now I think I misunderstood something in R or bnlearn package. Could you please tell me how to realize this task with filling the column by multiple cpqueries? That would help me a lot with my research!
cpquery is quite difficult to work with programmatically. If you look at the examples in the help page you can see the author uses eval(parse(...)) to build the queries. I have added two approaches below, one using the methods from the help page and one using cpdist to draw samples and reweighting to get the probabilities. Your example library(bnlearn); library(dplyr) data(learning.test) baynet = hc(learning.test) fit = bn.fit(baynet, learning.test) sttbl = arc.strength(x = baynet, data = learning.test) sttbl = sttbl %>% mutate(prob = NA) %>% arrange(strength) This uses cpquery and the much maligned eval(parse(...)) -- this is the approach the the bnlearn author takes to do this programmatically in the ?cpquery examples. Anyway, # You want the evidence and event to be the same; in your question it is `1` # but for example using learning.test data we use 'a' state = "\'a\'" # note if the states are character then these need to be quoted event = paste(sttbl$from, "==", state) evidence = paste(sttbl$to, "==", state) # loop through using code similar to that found in `cpquery` set.seed(1) # to make sampling reproducible for(i in 1:nrow(sttbl)) { qtxt = paste("cpquery(fit, ", event[i], ", ", evidence[i], ",n=1e6", ")") sttbl$prob[i] = eval(parse(text=qtxt)) } I find it preferable to work with cpdist which is used to generate random samples conditional on some evidence. You can then use these samples to build up queries. If you use likelihood weighting (method="lw") it is slightly easier to do this programatically (and without evil(parse(...))). The evidence is added in a named list i.e. list(A='a'). # The following just gives a quick way to assign the same # evidence state to all the evidence nodes. evidence = setNames(replicate(nrow(sttbl), "a", simplify = FALSE), sttbl$to) # Now loop though the queries # As we are using likelihood weighting we need to reweight to get the probabilities # (cpquery does this under the hood) # Also note with this method that you could simulate from more than # one variable (event) at a time if the evidence was the same. for(i in 1:nrow(sttbl)) { temp = cpdist(fit, sttbl$from[i], evidence[i], method="lw") w = attr(temp, "weights") sttbl$prob2[i] = sum(w[temp=='a'])/ sum(w) } sttbl # from to strength prob prob2 # 1 A D -1938.9499 0.6186238 0.6233387 # 2 A B -1153.8796 0.6050552 0.6133448 # 3 C D -823.7605 0.7027782 0.7067417 # 4 B E -720.8266 0.7332107 0.7328657 # 5 F E -549.2300 0.5850828 0.5895373
How to create a loop that changes part of a column name in a data frame
I am trying to find Cronbach's Alpha for survey data containing a series of multi-item measures. Rather than have to manually write out every single multi-item measure, it looks like something a loop should be able to manage far more effectively, but it needs to change only part of the column name, according to the question number. The basic idea as it currently sits in my head would be... for (N in 4:22) { ytqN <- data.frame(YT_Data$QNa, YT_Data$QNb, YT_Data$QNc) alpha(ytqN) } The loop would then create new data frames for each multi item measure and run Cronbach's Alpha as it goes. This doesn't work though. :( ytq4 <- data.frame(YT_Data$Q4a, YT_Data$Q4b, YT_Data$Q4c) alpha(ytq4) ytq5 <- data.frame(YT_Data$Q5a, YT_Data$Q5b, YT_Data$Q5c) alpha(ytq5) ytq6 <- data.frame(YT_Data$Q6a, YT_Data$Q6b, YT_Data$Q6c) alpha(ytq6) ytq7 <- data.frame(YT_Data$Q7a, YT_Data$Q7b, YT_Data$Q7c) alpha(ytq7) ytq8 <- data.frame(YT_Data$Q8a, YT_Data$Q8b, YT_Data$Q8c) alpha(ytq8) ytq9 <- data.frame(YT_Data$Q9a, YT_Data$Q9b, YT_Data$Q9c) alpha(ytq9) ytq10 <- data.frame(YT_Data$Q10a, YT_Data$Q10b, YT_Data$Q10c) alpha(ytq10) ytq11 <- data.frame(YT_Data$Q11a, YT_Data$Q11b, YT_Data$Q11c) alpha(ytq11) ytq12 <- data.frame(YT_Data$Q12a, YT_Data$Q12b, YT_Data$Q12c) alpha(ytq12) ytq13 <- data.frame(YT_Data$Q13a, YT_Data$Q13b, YT_Data$Q13c) alpha(ytq13) ytq14 <- data.frame(YT_Data$Q14a, YT_Data$Q14b, YT_Data$Q14c) alpha(ytq14) ytq15 <- data.frame(YT_Data$Q15a, YT_Data$Q15b, YT_Data$Q15c) alpha(ytq15) ytq16 <- data.frame(YT_Data$Q16a, YT_Data$Q16b, YT_Data$Q16c) alpha(ytq16) ytq17 <- data.frame(YT_Data$Q17a, YT_Data$Q17b, YT_Data$Q17c) alpha(ytq17) ytq18 <- data.frame(YT_Data$Q18a, YT_Data$Q18b, YT_Data$Q18c) alpha(ytq18) ytq19 <- data.frame(8 - YT_Data$Q19a, YT_Data$Q19b, YT_Data$Q19c) # Reverse code Q19a alpha(ytq19) ytq20 <- data.frame(YT_Data$Q20a, YT_Data$Q20b, YT_Data$Q20c) alpha(ytq20) ytq21 <- data.frame(YT_Data$Q21a, YT_Data$Q21b, YT_Data$Q21c) alpha(ytq21) ytq22 <- data.frame(YT_Data$Q22a, YT_Data$Q22b, YT_Data$Q22c) alpha(ytq22) The desired results would be a single output containing all the Cronbach's Alphas for the multi item measures for questions 4-22 in the data set I am currently working on executed via a single piece of code, rather than have to go question by question.
It's easier to help if you include your data, but I guess this should work: alpha_list = list() for(N in 4:22){ ytq = data.frame(YT_Data[paste0("Q",N,"a")], YT_Data[paste0("Q",N,"b")], YT_Data[paste0("Q",N,"c")]) alpha_list[[N]] = alpha(ytq) } We are using paste0() to create the column names while looping on N. alpha_list will be a list with the results given by alpha()
For each possible permutation of factor levels, apply function and also name list of results
Improve the following code by rewriting to be more compact (a one-liner with alply or similar?) Also if it can be made more performant (if possible). I have a dataframe with several categorical variables, each with various number of levels. (Examples: T1_V4: B,C,E,G,H,N,S,W and T1_V7: A,B,C,D ) For any specific one of those categorical vars, I want to do the following: Construct all possible level-permutations e.g. using DescTools::Permn() Then for each level.perm in those level.perms... Construct a list of function results where we apply some function to level.perm (in my particular case, recode the factor levels using level.perms, then take as.numeric, then compute correlation wrt some numeric response variable) Finally, name that list with the corresponding string-concatenated values of level.perm (e.g. 'DBCA') Example at bottom for permutations of A,B,C,D Reproducible example at bottom: The following code does this, can you improve on it? (I tried alply) require(DescTools) level.perms <- Permn(levels(MyFactorVariable)) tmp <- with(df, apply( level.perms, 1, function(var.levels) { cor(MyResponseVariable, as.numeric(factor(MyFactorVariable, levels=var.levels))) }) ) names(tmp) <- apply(level.perms, 1, paste, collapse='') Example (for CategVar1 with levels A,B,C,D): ABCD BACD BCAD ACBD CABD CBAD BCDA ACDB 0.031423 0.031237 0.002338 0.002116 -0.026496 -0.026386 -0.008743 -0.009104 CADB CBDA ABDC BADC CDAB CDBA ADBC BDAC -0.037228 -0.037364 0.048423 0.048075 -0.048075 -0.048423 0.037364 0.037228 BDCA ADCB DABC DBAC DBCA DACB DCAB DCBA 0.009104 0.008743 0.026386 0.026496 -0.002116 -0.002338 -0.031237 -0.031423 Reproducible example using randomly-generated dataframe: set.seed(120) df = data.frame(ResponseVar = exp(runif(1000, 0,4)), CategVar1 = factor(sample(c('A','B','C','D'), 1000, replace=T)), CategVar2 = factor(sample(c('B','C','E','G','H','N'), 1000, replace=T)) ) cor(as.numeric(df$CategVar1), df$MyResponseVar) # 0.03142 cor(as.numeric(df$CategVar2), df$MyResponseVar) # 0.02112 #then if you run the above code you get the above table of correlation values
Huge data file and running multiple parameters and memory issue, Fisher's test
I have a R code that I am trying to run in a server. But it is stopping in the middle/get frozen probably because of memory limitation. The data files are huge/massive (one has 20 million lines) and if you look at the double for loop in the code, length(ratSplit) = 281 and length(humanSplit) = 36. The data has specific data of human and rats' genes and human has 36 replicates, while rat has 281. So, the loop is basically 281*36 steps. What I want to do is to process data using the function getGeneType and see how different/independent are the expression of different replicate combinations. Using Fisher's test. The data rat_processed_7_25_FDR_05.out looks like this : 2 Sptbn1 114201107 114200202 chr14|Sptbn1:114201107|Sptbn1:114200202|reg|- 2 Thymus_M_GSM1328751 reg 2 Ndufb7 35680273 35683909 chr19|Ndufb7:35680273|Ndufb7:35683909|reg|+ 2 Thymus_M_GSM1328751 rev 2 Ndufb10 13906408 13906289 chr10|Ndufb10:13906408|Ndufb10:13906289|reg|- 2 Thymus_M_GSM1328751 reg 3 Cdc14b 1719665 1719190 chr17|Cdc14b:1719665|Cdc14b:1719190|reg|- 3 Thymus_M_GSM1328751 reg and the data fetal_output_7_2.out has the form SPTLC2 78018438 77987924 chr14|SPTLC2:78018438|SPTLC2:77987924|reg|- 11 Fetal_Brain_408_AGTCAA_L006_R1_report.txt reg EXOSC1 99202993 99201016 chr10|EXOSC1:99202993|EXOSC1:99201016|rev|- 5 Fetal_Brain_408_AGTCAA_L006_R1_report.txt reg SHMT2 57627893 57628016 chr12|SHMT2:57627893|SHMT2:57628016|reg|+ 8 Fetal_Brain_408_AGTCAA_L006_R1_report.txt reg ZNF510 99538281 99537128 chr9|ZNF510:99538281|ZNF510:99537128|reg|- 8 Fetal_Brain_408_AGTCAA_L006_R1_report.txt reg PPFIBP1 27820253 27824363 chr12|PPFIBP1:27820253|PPFIBP1:27824363|reg|+ 10 Fetal_Brain_408_AGTCAA_L006_R1_report.txt reg Now I have few questions on how to make this more efficient. I think when I run this code, R takes up lots of memory that ultimately causes problems. I am wondering if there is any way of doing this more efficiently Another possibility is the usage of double for-loop'. Will sapply help? In that case, how should I apply sapply? At the end I want to convert result into a csv file. I know this is a bit overwhelming to put code like this. But any optimization/efficient coding/programming will be A LOT! I really need to run the whole thing at least one to get the data soon. #this one compares reg vs rev date() ratRawData <- read.table("rat_processed_7_25_FDR_05.out",col.names = c("alignment", "ratGene", "start", "end", "chrom", "align", "ratReplicate", "RNAtype"), fill = TRUE) humanRawData <- read.table("fetal_output_7_2.out", col.names = c("humanGene", "start", "end", "chrom", "alignment", "humanReplicate", "RNAtype"), fill = TRUE) geneList <- read.table("geneList.txt", col.names = c("human", "rat"), sep = ',') #keeping only information about gene, alignment number, replicate and RNAtype, discard other columns ratRawData <- ratRawData[,c("ratGene", "ratReplicate", "alignment", "RNAtype")] humanRawData <- humanRawData[, c( "humanGene", "humanReplicate", "alignment", "RNAtype")] #function to capitalize capitalize <- function(x){ capital <- toupper(x) ## capitalize paste0(capital) } #capitalizing the rna type naming for rat. So, reg ->REG, dup ->DUP, rev ->REV #doing this to make data manipulation for making contingency table easier. levels(ratRawData$RNAtype) <- capitalize(levels(ratRawData$RNAtype)) #spliting data in replicates ratSplit <- split(ratRawData, ratRawData$ratReplicate) humanSplit <- split(humanRawData, humanRawData$humanReplicate) print("done splitting") #HyRy :when some gene has only reg, rev , REG, REV #HnRy : when some gene has only reg,REG,REV #HyRn : add 1 when some gene has only reg,rev,REG #HnRn : add 1 when some gene has only reg,REG #function to be used to aggregate getGeneType <- function(types) { types <- as.character(types) if ('rev' %in% types) { return(ifelse(('REV' %in% types), 'HyRy', 'HyRn')) } else { return(ifelse(('REV' %in% types), 'HnRy', 'HnRn')) } } #logical function to see whether x is integer(0) ..It's used the for loop bellow in case any one HmYn is equal to zero is.integer0 <- function(x) { is.integer(x) && length(x) == 0L } result <- data.frame(humanReplicate = "human_replicate", ratReplicate = "rat_replicate", pvalue = "p-value", alternative = "alternative_hypothesis", Conf.int1 = "conf.int1", Conf.int2 ="conf.int2", oddratio = "Odd_Ratio") for(i in 1:length(ratSplit)) { for(j in 1:length(humanSplit)) { ratReplicateName <- names(ratSplit[i]) humanReplicateName <- names(humanSplit[j]) #merging above two based on the one-to-one gene mapping as in geneList defined above. mergedHumanData <-merge(geneList,humanSplit[[j]], by.x = "human", by.y = "humanGene") mergedRatData <- merge(geneList, ratSplit[[i]], by.x = "rat", by.y = "ratGene") mergedHumanData <- mergedHumanData[,c(1,2,4,5)] #rearrange column mergedRatData <- mergedRatData[,c(2,1,4,5)] #rearrange column mergedHumanRatData <- rbind(mergedHumanData,mergedRatData) #now the columns are "human", "rat", "alignment", "RNAtype" agg <- aggregate(RNAtype ~ human+rat, data= mergedHumanRatData, FUN=getGeneType) #agg to make HmYn form HmRnTable <- table(agg$RNAtype) #table of HmRn ie RNAtype in human and rat. #now assign these numbers to variables HmYn. Consider cases when some form of HmRy is not present in the table. That's why #is.integer0 function is used HyRy <- ifelse(is.integer0(HmRnTable[names(HmRnTable) == "HyRy"]), 0, HmRnTable[names(HmRnTable) == "HyRy"][[1]]) HnRn <- ifelse(is.integer0(HmRnTable[names(HmRnTable) == "HnRn"]), 0, HmRnTable[names(HmRnTable) == "HnRn"][[1]]) HyRn <- ifelse(is.integer0(HmRnTable[names(HmRnTable) == "HyRn"]), 0, HmRnTable[names(HmRnTable) == "HyRn"][[1]]) HnRy <- ifelse(is.integer0(HmRnTable[names(HmRnTable) == "HnRy"]), 0, HmRnTable[names(HmRnTable) == "HnRy"][[1]]) contingencyTable <- matrix(c(HnRn,HnRy,HyRn,HyRy), nrow = 2) # contingencyTable: # HnRn --|--HyRn # |------|-----| # HnRy --|-- HyRy # fisherTest <- fisher.test(contingencyTable) #make new line out of the result of fisherTest newLine <- data.frame(t(c(humanReplicate = humanReplicateName, ratReplicate = ratReplicateName, pvalue = fisherTest$p, alternative = fisherTest$alternative, Conf.int1 = fisherTest$conf.int[1], Conf.int2 =fisherTest$conf.int[2], oddratio = fisherTest$estimate[[1]]))) result <-rbind(result,newLine) #append newline to result if(j%%10 = 0) print(c(i,j)) } } write.table(result, file = "compareRegAndRev.csv", row.names = FALSE, append = FALSE, col.names = TRUE, sep = ",")
Referring to the accepted answer to Monitor memory usage in R, the amount of memory used by R can be tracked with gc(). If the script is, indeed, running short of memory (which would not surprise me), the easiest way to resolve the problem would be to move the write.table() from the outside to the inside of the loop, to replace the rbind(). It would just be necessary to create a new file name for the CSV file that is written from each output, e.g. by: csvFileName <- sprintf("compareRegAndRev%03d_%03d.csv",i,j) If the CSV files are written without headers, they could then be concatenated separately outside R (e.g. using cat in Unix) and the header added later. While this approach might succeed in creating the CSV file that is sought, it is possible that file might be too big to process subsequently. If so, it may be preferable to process the CSV files individually, rather than concatenating them at all.
Extract a predictors form constparty object (CHAID output) in R
I have a large dataset (questionnaire results) of mostly categorical variables. I have tested for dependency between the variables using chi-square test. There are incomprehensible number of dependencies between variables. I used the chaid() function in the CHAID package to detect interactions and separate out (what I hope to be) the underlying structure of these dependencies for each variable. What typically happens is that the chi-square test will reveal a large number of dependencies (say 10-20) for a variable and the chaid function will reduce this to something much more comprehensible (say 3-5). What I want to do is to extract the names of those variable that were shown to be relevant in the chaid() results. The chaid() output is in the form of a constparty object. My question is how to extract the variable names associated with the nodes in such an object. Here is a self contained code example: library(evtree) # for the ContraceptiveChoice dataset library(CHAID) library(vcd) library(MASS) data("ContraceptiveChoice") longform = formula(contraceptive_method_used ~ wifes_education + husbands_education + wifes_religion + wife_now_working + husbands_occupation + standard_of_living_index + media_exposure) z = chaid(longform, data = ContraceptiveChoice) # plot(z) z # This is the part I want to do programatically shortform = formula(contraceptive_method_used ~ wifes_education + husbands_occupation) # The thing I want is a programatic way to extract 'shortform' from 'z' # Examples of use of 'shortfom' loglm(shortform, data = ContraceptiveChoice)
One possible sollution: nn <- nodeapply(z) n.names= names(unlist(nn[[1]])) ext <- unlist(sapply(n.names, function(x) grep("split.varid.", x, value=T))) ext <- gsub("kids.split.varid.", "", ext) ext <- gsub("split.varid.", "", ext) dep.var <- as.character(terms(z)[1][[2]]) # get the dependent variable plus = paste(ext, collapse=" + ") mul = paste(ext, collapse=" * ") shortform <- as.formula(paste (dep.var, plus, sep = " ~ ")) satform <- as.formula(paste (dep.var, mul, sep = " ~ ")) mosaic(shortform, data = ContraceptiveChoice) #stp <- step(glm(satform, data=ContraceptiveChoice, family=binomial), direction="both")