How to manually build predictions from xgboost model - r

Question

I am trying to figure out how to generate probabilities from the xgboost model tree so they match what I would get from the predict function.
First I build the model
library(xgboost)
#install.packages("ModelMetrics")
library(ModelMetrics)
set.seed(100)
# - Extreme gbm
y = as.integer(testDF$y)
x = testDF[,-which(names(testDF) %in% c('y'))]
var.names <- names(x)
x = as.matrix(x)
x = matrix(as.numeric(x),nrow(x),ncol(x))
nround = 10
XX <- xgboost(param=param, data = x, label = y, nrounds=nround, missing = NA)
Then I wrote some code to build all of the rules that would result in a particular leaf
baseTree <- xgb.model.dt.tree(model = XX)
Leafs <- filter(baseTree, Feature == 'Leaf')
Branches <- filter(baseTree, Feature != 'Leaf')
Branches$Feature = var.names[as.numeric(Branches$Feature) + 1]
FullRules = rep(NA, nrow(Leafs))
AllRules <- foreach(i = 1:nrow(Leafs), .combine = 'rbind') %do% {
theLeaf = Leafs[i,]
theNode = theLeaf$Node
theID = theLeaf$ID
count = 1
RuleText = ''
while(theNode != 0){
FF <- filter(Branches, Yes == theID | No == theID | Missing == theID)
isYes = FF$Yes == theID
isNo = FF$No == theID
isMissing = FF$Missing == theID
FullRules[i] = ifelse(isYes & isMissing
, paste0("(", FF$Feature, " < ", FF$Split, " | is.na(", FF$Feature, "))")
, NA)
FullRules[i] = ifelse(isNo & isMissing
, paste0("(", FF$Feature, " >= ", FF$Split, " | is.na(", FF$Feature, "))")
, FullRules[i])
FullRules[i] = ifelse(isYes & !isMissing
, paste0(FF$Feature, " < ", FF$Split)
, FullRules[i])
FullRules[i] = ifelse(isNo & !isMissing
, paste0(FF$Feature, " >= ", FF$Split)
, FullRules[i])
FullRules[i] = ifelse(isMissing & !isYes & !isNo
, paste0("is.na(", FF$Feature, ")")
, FullRules[i])
if(count == 1){
RuleText = FullRules[i]
} else{
RuleText = paste0(RuleText, " & ", FullRules[i])
}
theNode = FF$Node
theID = FF$ID
count = count + 1
}
data.frame(
Leafs[i,]
,RuleText
)
}
Now I pick out 1 row and attempted to match the probabilities. In this case it matches. The loop will go through and indicate TRUE for all of the rules that are met for this particular customer. Then I can filter down to those rows and sum those up to get the logodds estimates. Then I convert those to probabilities.
TT <- testDF[25,]
ff <- foreach(i = 1:nrow(AllRules), .combine = 'rbind') %do% {
TT %>% transmute_(
Tree = as.character(AllRules$RuleText[i])
, Quality = AllRules$Quality[i])
}
predict(XX, as.matrix(TT[,var.names]))
#[1] 0.05571342
filter(ff, Tree) %>%
summarise(
Q1 = sum(sqrt(Quality^2))
# ,Q2 = sum(sqrt(Quality^2))
, Prob1 = exp(Q1)/(1+exp(Q1))
, Prob2 = 1-Prob1
)
# Q1 Prob1 Prob2
#1 2.830209 0.9442866 0.0557134
But in this case it does not match the predict function...
TT <- testDF[17,]
ff <- foreach(i = 1:nrow(AllRules), .combine = 'rbind') %do% {
TT %>% transmute_(
Tree = as.character(AllRules$RuleText[i])
, Quality = AllRules$Quality[i])
}
predict(XX, as.matrix(TT[,var.names]))
#[1] 0.1386877
filter(ff, Tree) %>%
summarise(
Q1 = sum(sqrt(Quality^2))
# ,Q2 = sum(sqrt(Quality^2))
, Prob1 = exp(Q1)/(1+exp(Q1))
, Prob2 = 1-Prob1
)
# Q1 Prob1 Prob2
#1 1.967608 0.877354 0.122646

Answer 1

To generate the prediction you just need to sum up the values of the individual leafs that the person falls within for each booster
filter(ff, Tree) %>%
summarise(
Q1 = sum(Quality)
, Prob1 = exp(Q1)/(1+exp(Q1))
, Prob2 = 1-Prob1
)