SparkR: Generate Density Data From Numeric Columns - r

Question

I am wondering how I could generate a data object like the one you get when calling stats::density(df$variable) on the numeric columns of a spark dataframe?
I am looking into SparkR::spark.lapply but think I am missing something. I have created a little example below. If someone knows how and is willing to help me out, I'd be very thankful.
Best,
NF
Example:
df<- iris
gen_density_data<- function(df){
col_types<- sapply(df, class)
good_cols<- which(col_types %in% c("numeric", "integer"))
tres<- lapply(good_cols, function(x){
expr<- paste0("stats::density(df$", colnames(df)[x], ")")
eval(parse(text=expr))
})
return(tres)
}
res<- gen_density_data(df)
# And for Spark:
sdf<- SparkR::createDataFrame(iris)
gen_spark_density_data<- function(sdf){
tmp_types<- SparkR::coltypes(sdf)
good_cols_idx<- which(tmp_types %in% setdiff(tmp_types, c("character", "POSIXct", "POSIXlt", "logical")))
if(length(good_cols_idx)>=1){
tres<- SparkR::spark.lapply(good_cols_idx, function(x){
eval(parse(text=paste0("stats::density(sdf$", colnames(sdf)[x], ")")))
})
return(tres)
}
}
tst<- gen_spark_density_data(sdf=sdf) # This is where it throws errors.

Answer 1

I came up with a solution that works decently well. I use highcharter for plotting. I think I could further improve how I manage the partitions of the data. Right now, this might not be the most scalable solution for large data sets with columns with large differences between the minimum and maximum values. Some conditional checking is probably in order, but for the purposes of just getting an example out there, this is what I made. Note: I adopted the example from https://rpubs.com/mcocam12/KDF_byHand. Many thanks to Marc for the example.
Data:
df<- do.call("rbind", replicate(10, iris, simplify = FALSE))
sdf<- SparkR::createDataFrame(df)
sdf<- SparkR::repartition(sdf, nrow(sdf))
Functions:
gen_sdf_kernel_density_points<- function(sdf=sdf,num_values, h=1){
x_col<- SparkR::colnames(sdf)[1]
min_max_sdf<- eval(parse(text=paste0("SparkR::agg(sdf, min=min(sdf$", x_col, "), max=max(sdf$", x_col,")) %>% SparkR::collect()")))
Range = seq(min_max_sdf$min-5, min_max_sdf$max+5, 0.01)
Range<- data.frame(Range)
RangeSDF<- SparkR::createDataFrame(Range)
# this is where I think I could be better with partitions, ideas welcomed
#RangeSDF<- SparkR::repartition(RangeSDF, nrow(RangeSDF))
# if(nrow(Range)>1000){
# RangeSDF<- SparkR::repartition(RangeSDF, 200L)
# } else if(nrow(Range) > 64){
# RangeSDF<- SparkR::repartition(RangeSDF, 64L)
# }
tst<- SparkR::crossJoin(sdf, RangeSDF)
tst$density<- eval(parse(text=paste0("exp(-(tst$Range-tst$", x_col,")^2/(2*h^2))/(h*sqrt(2*pi))")))
## Now group by range and get the sum of the density, normalize by the number of values
gb_df<- SparkR::groupBy(tst, tst$Range)
densities2<- SparkR::agg(gb_df, bell_sum=sum(tst$density))
densities2<- SparkR::withColumn(densities2, "kernel_density", densities2$bell_sum / num_values)
densities2<- SparkR::arrange(densities2, asc(densities2$Range))
return(densities2)
}
gen_den_plots_from_spark_res<- function(res){
big_out<- lapply(seq_along(res), function(x){
var_name<- names(res)[x]
rdf<- res[[x]]
tmp<- data.frame(cbind(x = rdf$Range, y = rdf$kernel_density))
x<- highcharter::list_parse(tmp)
hc<- highcharter::highchart() %>%
hc_series(
list(
name="Density Estimate",
data = x,
type = "areaspline",
marker = list(enabled = FALSE),
color = list(
linearGradient = list(x1 = 0, y1 = 1, x2 = 0, y2 = 0),
stops = list(
list(0, "transparent"),
list(0.33, "#0000FF1A"),
list(0.66, "#0000FF33"),
list(1, "#ccc")
)
),
fillColor = list(
linearGradient = list(x1 = 0, y1 = 1, x2 = 0, y2 = 0),
stops = list(
list(0, "transparent"),
list(0.1, "#0000FF1A"),
list(0.5, "#0000FF33"),
list(1, "#0000FF80")
)
)
)
)
hc<- hc %>%
highcharter::hc_title(text=paste0("Density Plot For: ", snakecase::to_title_case(var_name)))# %>% hc_add_series(data =tmp, hcaes(x= tmp$x, y = tmp$y),name="Bars", type="column" )
return(hc)
})
return(big_out)
}
make_hc_grid<- function(tres_out, ncol=2){
hc<- tres_out %>%
highcharter::hw_grid(rowheight = 450, ncol = ncol) %>%htmltools::browsable()
hc
}
Usage:
tmp_types<- SparkR::coltypes(sdf)
good_cols_idx<- which(tmp_types %in% setdiff(tmp_types, c("character", "POSIXct", "POSIXlt", "logical")))
nrows_sdf<- SparkR::count(sdf)
if(length(good_cols_idx)>=1){
out<- lapply(seq_along(good_cols_idx), function(z){
# Need to select a single column for the sdf, otherwise the cross join becomes too big
tmpz<- SparkR::select(sdf, SparkR::colnames(sdf)[good_cols_idx[z]])
out<- gen_sdf_kernel_density_points(sdf = tmpz, num_values = nrows_sdf)
out<- SparkR::collect(out)
return(out)
}) %>% stats::setNames(SparkR::colnames(sdf)[good_cols_idx])
}
Plotting:
tres<- gen_den_plots_from_spark_res(res=out)
all_plots<- make_hc_grid(tres_out = tres)
# View Result
all_plots
Expected Result:
This could all probably be improved...if you have ideas, I'd love to hear them.
Best,
NF