Loop over data.frame and display result immediately in DT - r

Question

I have a simple shiny app that holds a dataset as a reactive value.
Once a button is pressed, a function should be applied to each row and the result is added as another variable to that dataset.
The dataset is also shown as a DT.
The result variable should be rendered as soon as the computation for that row is finished.
At the moment, the loop/apply that applies the function to each row finishes and only afterwards the results are displayed.
As the function can run for a long time, I want the DT to be updated as soon as a run is finished, not when all runs finish.
I understand that this means I need to use promises/future so that the main shiny code block spawns new processes which do not block in this case the main thread from updating the values. Correct?
However, I am not able to get it to work.
Here is a small MWE using a simple for loop
library(shiny)
library(DT)
ui <- fluidPage(
actionButton("run", "RUN"),
hr(),
DT::dataTableOutput("table")
)
calc_fun <- function(val) {
Sys.sleep(0.5)
val * 10
}
server <- function(input, output, session) {
set.seed(123)
data_res <- reactiveVal(data.frame(id = 1:10, val = rnorm(10), val10 = NA))
observe({
for (i in seq(nrow(data_res()))) {
print(paste("Looking at row", i))
d <- data_res()
d[i, "val10"] <- calc_fun(val = d[i, "val"])
data_res(d)
}
}) %>% bindEvent(input$run)
# This should be rendered whenever a round in the for-loop has finished
# at the moment it is only run once the loop is finished
output$table <- DT::renderDataTable(data_res())
}
shinyApp(ui, server)

Answer 1

Thanks to #ismirsehregal, I came up with the following solution which uses futures to start the calculation in the background, which in turn write the current status to a file.
Shiny then reactively reads the file and updates the values.
The full MWE looks like this:
library(shiny)
library(DT)
library(future)
library(promises)
library(qs) # for fast file read/write, replace with csv if needed
plan(multisession)
ui <- fluidPage(
actionButton("run", "RUN"),
hr(),
textOutput("prog"),
uiOutput("status"),
hr(),
fluidRow(
column(6,
h2("Current Status"),
DT::dataTableOutput("table")
),
column(6,
h2("Data in File"),
tableOutput("file_data")
)
)
)
calc_fun <- function(val) {
Sys.sleep(runif(1, 0, 2))
val * 10
}
# main function that goes through the rows and starts the calculation
# note that the output is saved to a .qs file to be read in by another reactive
do_something_per_row <- function(df, outfile) {
out <- tibble(id = numeric(0), res = numeric(0))
for (i in seq(nrow(df))) {
v <- df$val[i]
out <- out %>% add_row(id = i, res = calc_fun(v))
qsave(out, outfile)
}
return(out)
}
# create a data frame of tasks
set.seed(123)
N <- 13
tasks_init <- tibble(id = seq(N), val = round(rnorm(N), 2), status = "Open", res = NA)
server <- function(input, output, session) {
# the temporary file to communicate over
outfile <- "temp_progress_watch.qs"
unlink(outfile)
data <- reactiveVal(tasks_init) # holds the current status of the tasks
data_final <- reactiveVal() # holds the results once all tasks are finished
output$prog <- renderText(sprintf("Progress: 0 of %i (0.00%%)", nrow(data())))
output$status <- renderUI(div(style = "color: black;", h3("Not yet started")))
# on the button, start the do_something_per_row function as a future
observeEvent(input$run, {
# if a file exists => the code runs already
if (file.exists(outfile)) return()
print("Starting to Run the code")
output$status <- renderUI(div(style = "color: orange;", h3("Working ...")))
d <- data()
future({do_something_per_row(d, outfile)}, seed = TRUE) %...>% data_final()
print("Done starting the code, runs now in the background! freeing the session for interaction")
# return(NULL) # hide future
})
observe({
req(data_final())
output$status <- renderUI(div(style = "color: green;", h3("Done")))
print("All Done - Results came back from the future!")
})
output$file_data <- renderTable(req(df_done()))
output$table <- DT::renderDataTable({
# no need to fire on every refresh, this is handled automatically later
DT::datatable(isolate(data())) %>%
formatStyle("status", color = styleEqual(c("Open", "Done"), c("white", "black")),
backgroundColor = styleEqual(c("Open", "Done"), c("red", "green")))
})
dt_proxy <- DT::dataTableProxy("table")
# look for changes in the file and load it
df_done <- reactiveFileReader(300, session, outfile, function(f) {
r <- try(qread(f), silent = TRUE)
if (inherits(r, "try-error")) return(NULL)
r
})
observe({
req(df_done())
open_ids <- data() %>% filter(status == "Open") %>% pull(id)
if (!any(df_done()$id %in% open_ids)) return()
print(paste("- new entry found:", paste(intersect(df_done()$id, open_ids), collapse = ", ")))
rr <- data() %>% select(-res) %>% left_join(df_done(), by = "id") %>%
mutate(status = ifelse(is.na(res), "Open", "Done"))
data(rr)
DT::replaceData(dt_proxy, rr)
# replace the progress text
txt <- sprintf("Progress: % 4i of % 4i (%05.2f%%)",
nrow(df_done()), nrow(data()), 100 * (nrow(df_done()) / nrow(data())))
output$prog <- renderText(txt)
})
}
shinyApp(ui, server)
or as a picture: