Writing Custom Layer (3D RoI Pooling) using Keras in R - r

Question

Using R, I would like to produce a Keras custom layer that performs 3D Region of Interest (RoI) Pooling. My 3D RoI Pooling function works as expected outside of an R6Class KerasLayer, but I am having trouble integrating it into a Custom Layer. I am not sure if I am using autograph correctly in the below example. I am under the impressing that for me to use the R functionalities (i.e. for loop) in the "call" section of the custom layer I am required to wrap the function in autograph.
Is the below custom layer for applying a 3D RoI Pooling (without training weights) appropriate syntex for an R user of Keras?
The input for the 3d_RoI_Pool layer includes:
(i) output layer from a VNet model,
shape=(None, 16, 16, 40, 1)
(ii) tensor defining the "index" for cropping the input layer to the extent of each RoI,
- tf.Variable 'Variable:0' shape=(1, 1, 6) dtype=float64, numpy=array([[[ 5., 5., 4., 5., 10., 20.]]])
(iii) output dimension for the pooled 3D RoIs.
In this case its c(1, 1, 5, 5, 5, 1) for c(Batch_Size, RoI_Size, X, Y, Z, Channels)
Below are some questions that I am unclear about:
I initially apply layer_cropping_3d to crop each RoI but am not sure if custom layer is allowed to use available R Keras layers? For this reason, in code I performed my own cropping.
Is it the correct use of autograph within custom layer?
Should I instead use the layer_lambda() for such a function (i.e. Wraps arbitrary expression as a layer), given that I do not have trainable weights in the 3D_RoI_Pooling layer?
Below is the Code that gets the output layer from a V-Net CNN and prepares inputs for the custom layer:
# FINAL LAYERS OF V_Net
Output_shortcut = up1_DeConv
Final_Conv <- up1_Concat %>%
# 1st
layer_conv_3d(filters = Filter_Start, kernel_size = Kernel_Conv_Size, padding = "same", name = "Final_Conv") %>%
layer_batch_normalization() %>%
layer_activation("relu")
Final_Conv <- layer_add(c(Final_Conv, Output_shortcut), name = "Final_Conv_ResNet")
# PREPARE INPUT FOR CUSTOM LAYER
n_RoI <- 1
Batch_Size <- 1
Table_RoI <- list(list(c(5, 5), c(4, 5), c(10, 20)))
Table_RoI <- array_reshape(unlist(Table_RoI), c(Batch_Size,n_RoI,6), order="F")
Table_RoI <- tf$Variable(Table_RoI, tf$int16)
output_dim <- c(Batch_Size, n_RoI, 5,5,5, as.numeric(dim(Final_Conv)[5]))
# FEED CUSTOM LAYER TO CNN
Final_Conv2 <- Final_Conv %>% layer_3D_ROI_pooled(Table_RoI=Table_RoI, output_dim= output_dim)
Below is the actual 3D RoI Pooling Custom Layer that generates an error:
# 3D ROIpooled_Layer (R KERAS )
ROIpooled_Layer <- R6::R6Class("KerasLayer",
inherit = KerasLayer,
public = list(
Table_RoI = NULL,
output_dim = NULL,
###############
# INITIALISE
###############
initialize = function(Table_RoI, output_dim) {
self$Table_RoI = Table_RoI
self$output_dim = output_dim
},
###############
# CALL FUNCTION ROIpooled_Function <-
###############
call = autograph(function(x, mask = NULL,
Table_RoI,
output_dim) {
## Input_L ROI_Table
n_Batch <- output_dim[1]
n_RoI <- output_dim[2]
Channels <- output_dim[6]
for(r in 1:n_RoI){
# layer_cropping_3d(Input_L, cropping = list(list(as.numeric(Table_RoI[,r,1]), as.numeric(Table_RoI[,r,2])),
# list(as.numeric(Table_RoI[,r,3]), as.numeric(Table_RoI[,r,4])),
# list(as.numeric(Table_RoI[,r,5]), as.numeric(Table_RoI[,r,6]))))
RoI_Cropped <-x[,(as.numeric(Table_RoI[,r,1])+1):(dim(feature_map_T)[2]-as.numeric(Table_RoI[,r,2])),
(as.numeric(Table_RoI[,r,3])+1):(dim(feature_map_T)[3]-as.numeric(Table_RoI[,r,4])),
(as.numeric(Table_RoI[,r,5])+1):(dim(feature_map_T)[4]-as.numeric(Table_RoI[,r,6])),]
RoI_X_Res <- as.array(k_shape(RoI_Cropped)[2])
RoI_Y_Res <- as.array(k_shape(RoI_Cropped)[3])
RoI_Z_Res <- as.array(k_shape(RoI_Cropped)[4])
New_X_Res <- as.array(output_dim[3])
New_Y_Res <- as.array(output_dim[4])
New_Z_Res <- as.array(output_dim[5])
X_step = RoI_X_Res / New_X_Res
Y_step = RoI_Y_Res / New_Y_Res
Z_step = RoI_Z_Res / New_Z_Res
for(ch in 1:Channels) {
print(paste("ch", ch))
for (k in 1:New_Z_Res) {
print(paste("k", k))
for (j in 1:New_Y_Res) {
print(paste("j", j, "k", k))
for (i in 1:New_X_Res) {
# INDEX X
Index_Xstart <- floor((i-1)*X_step+1)
if(i+1 <= RoI_X_Res){
Index_Xend <- floor((i)*X_step)
}else{
Index_Xend <- RoI_X_Res
}
# INDEX Y
Index_Ystart <- floor((j-1)*Y_step+1)
if(j+1 <= RoI_Y_Res){
Index_Yend <- floor((j)*Y_step)
}else{
Index_Yend <-RoI_Y_Res
}
# INDEX Z
Index_Zstart <- floor((k-1)*Z_step+1)
if(k+1 <= RoI_Z_Res){
Index_Zend <- floor((k)*Z_step)
}else{
Index_Zend <-RoI_Z_Res
}
Max_Pool_X_Value <- as.array(k_max(RoI_Cropped[n_Batch,Index_Xstart:Index_Xend, Index_Ystart:Index_Yend, Index_Zstart:Index_Zend,ch])) # ADD BATCH AND CHANNEL LAYERS
RoI_Pooled_Array[,r,i,j,k,ch] <- Max_Pool_X_Value
}# i LOOP
} # j LOOP
} # k Loop
} #Ch LOOP
} # r LOOP (ROI)
feature_map_ROIpooled <- tf$Variable(RoI_Pooled_Array, tf$int16) # ??? NOT SURE IF RETURN NEEDS TO BE A VARIABLE
return (feature_map_ROIpooled)
}), # END OF AUTO, # END OF CALL ... AUTOGRAPH FUNCTION
##############
# OUTPUT SHAPE
##############
compute_output_shape = function(input_shape) {
list(self$output_dim)
}
)
)
#
###############################
# Create layer wrapper function
###############################
layer_3D_ROI_pooled <- function(object, Table_RoI, output_dim, name = NULL, trainable = TRUE) {
create_layer(ROIpooled_Layer, object, list(Table_RoI = Table_RoI,
output_dim = as.integer(output_dim),
name = name,
trainable = FALSE
))
}
ERROR:
Error in value[[3L]](cond) :
The R function's signature must not contains esoteric Python-incompatible constructs. Detailed traceback: SyntaxError: non-default argument follows default argument (<string>, line 3)
Any help/insight/clarity would be appreciated.
Kind Regards,
Dom

Answer 1

I'm not sure if this is the correct protocol to answer own question but I think I have a working 3D RoI pooling Custom Layer to share. Many mistakes in above but most notable change is in the for loop.... I think I needed to first generate a list of tensors that represent each pooled RoI and then re-shape it to the desired output shape.
################################################################################################
# ROI_3D_pooled_Layer (Custom layer class)
##########################################
ROI_3D_pooled_Layer <- R6::R6Class("KerasLayer",
inherit = KerasLayer,
public = list(
List_RoI = NULL,
output_dim = NULL,
initialize = function(List_RoI, output_dim) {
self$List_RoI = List_RoI
self$output_dim = output_dim
},
call = function(x, mask = NULL) {
List_RoI <- self$List_RoI
output_dim <- self$output_dim
# EXTRACT INFORMATION ON OUTPUT DIMENSION
n_Batch <- as.integer(output_dim[1])
n_RoIs <- as.integer(output_dim[2])
n_Channels <- as.integer(output_dim[6])
New_X_Res <- as.numeric(output_dim[3])
New_Y_Res <- as.numeric(output_dim[4])
New_Z_Res <- as.numeric(output_dim[5])
input_shape <- dim(x)
# EMPTY LIST TO STORE TENSORS
output_list = list()
for(r in 1:n_RoIs){ # LOOP RoIs
# GET one RoI AND CROP INPUT LAYER
if(n_RoIs > 1){
oneList_RoI <- List_RoI[[r]]
}else{
oneList_RoI <- List_RoI
}
RoI_Cropped <-x[,(oneList_RoI[[1]][1]+1):(as.numeric(input_shape[2])-oneList_RoI[[1]][2]),
(oneList_RoI[[2]][1]+1):(as.numeric(input_shape[3])-oneList_RoI[[2]][2]),
(oneList_RoI[[3]][1]+1):(as.numeric(input_shape[4])-oneList_RoI[[3]][1]),]
# GET RoI Dimensions for XYZ
RoI_X_Res <- as.numeric(dim(RoI_Cropped)[2])
RoI_Y_Res <- as.numeric(dim(RoI_Cropped)[3])
RoI_Z_Res <- as.numeric(dim(RoI_Cropped)[4])
# CALCULATE STEPS IN ALL DIMENSIONS FOR POOLING
X_step = RoI_X_Res / New_X_Res
Y_step = RoI_Y_Res / New_Y_Res
Z_step = RoI_Z_Res / New_Z_Res
for(ch in 1:n_Channels) { # LOOP CHANNEL
for (k in 1:New_Z_Res) { # LOOP Z
for (j in 1:New_Y_Res) { # LOOP Y
for (i in 1:New_X_Res) { # LOOP X
# INDEX X
Index_Xstart <- floor((i-1)*X_step+1)
if(i+1 <= RoI_X_Res){
Index_Xend <- floor((i)*X_step)
}else{
Index_Xend <- RoI_X_Res
}
# INDEX Y
Index_Ystart <- floor((j-1)*Y_step+1)
if(j+1 <= RoI_Y_Res){
Index_Yend <- floor((j)*Y_step)
}else{
Index_Yend <-RoI_Y_Res
}
# INDEX Z
Index_Zstart <- floor((k-1)*Z_step+1)
if(k+1 <= RoI_Z_Res){
Index_Zend <- floor((k)*Z_step)
}else{
Index_Zend <-RoI_Z_Res
}
# MAX POOL VOLUME FOR EACH ELEMENT IN FINAL LAYER AND PUT IN EMPTY ARRAY
Max_Pool_X_Value <-k_max(RoI_Cropped[,Index_Xstart:Index_Xend, Index_Ystart:Index_Yend, Index_Zstart:Index_Zend,ch])
# APPEND EACH RoI_Pooled element into a list
output_list <- list.append(output_list, Max_Pool_X_Value)
}# i LOOP
} # j LOOP
} # k Loop
} #Ch LOOP
} # r LOOP (ROI)
# STACK THE OUTPUT LIST AND RESHAPE TO THE DESIRED OUTPUT SIZE
output_Stack <- k_stack(output_list, axis = 1)
feature_map_ROIpooled <- k_reshape(output_Stack, shape = c(n_Batch, n_RoIs, New_X_Res, New_Y_Res, New_Z_Res, n_Channels))
return (feature_map_ROIpooled)
},
compute_output_shape = function(input_shape) {
return(self$output_dim)
}
)
)
# Create layer wrapper function
layer_3D_ROI_pooled <- function(object, List_RoI, output_dim) {
create_layer(ROI_3D_pooled_Layer, object, list(List_RoI = List_RoI,
output_dim = as.integer(output_dim)
))
}
This is my first attempt with a Keras Custom Layer (using R) so please provide suggestions for improvement.