Should SEXP function args be PROTECTed when put inside an Rcpp::Xptr? - r

Look at the (oversimplified) Rcpp + R code below :
test.cpp :
#include <Rcpp.h>
using namespace Rcpp;
class VecWrap{
public:
SEXP vector;
int type;
VecWrap(SEXP vector)
{
this->vector = vector;
this->type = TYPEOF(vector);
if(this->type != INTSXP && this->type != REALSXP)
stop("invalid type");
}
bool contains(double val){
if(type == INTSXP){
IntegerVector v = vector;
for(int i = 0; i < v.size(); i++)
if(v[i] == val)
return true;
}else if(type == REALSXP){
NumericVector v = vector;
for(int i = 0; i < v.size(); i++)
if(v[i] == val)
return true;
}
return false;
}
};
// [[Rcpp::export]]
SEXP createVecWrap(SEXP x) {
VecWrap* w = new VecWrap(x);
return XPtr< VecWrap >(w);
}
// [[Rcpp::export]]
SEXP vecWrapContains(XPtr< VecWrap > w, double val){
return wrap(w->contains(val));
}
test.R :
library(Rcpp)
sourceCpp(file='test.cpp')
v <- 1:10e7
w <- createVecWrap(v)
vecWrapContains(w, 10000) # it works
# remove v and call the garbage collector
rm(v)
gc()
vecWrapContains(w, 10000) # R crashes (but it works with small vector "v")
Basically I put inside the custom class VecWrap the SEXP vector received as argument of createVecWrap function, in order to use it later.
But, as explained by the comments in the code, if I remove the vector v from the R-side and call the garbage collector, the R process crashes when I try to access the vector.
Should the vector be protected by the GC in someway ? If so, how? (Rcpp-style if possible)

Generally speaking you should try to stick to the C++ type system / Rcpp classes as much as possible (re: avoid handling SEXP directly if possible). However, the RObject class will provide your SEXP with protection from the garbage collector, and seems to work in this case:
#include <Rcpp.h>
class VecWrap {
public:
Rcpp::RObject vector;
int type;
VecWrap(SEXP vector_)
: vector(vector_)
{
type = vector.sexp_type();
if (type != INTSXP && type != REALSXP) {
Rcpp::stop("invalid type");
}
}
bool contains(double val) {
if (type == INTSXP){
Rcpp::IntegerVector v = Rcpp::as<Rcpp::IntegerVector>(vector);
for (int i = 0; i < v.size(); i++) {
if (v[i] == val) return true;
}
} else if (type == REALSXP) {
Rcpp::NumericVector v = Rcpp::as<Rcpp::NumericVector>(vector);
for (int i = 0; i < v.size(); i++) {
if (v[i] == val) return true;
}
}
return false;
}
};
// [[Rcpp::export]]
Rcpp::XPtr<VecWrap> createVecWrap(SEXP x) {
return Rcpp::XPtr<VecWrap>(new VecWrap(x));
}
// [[Rcpp::export]]
bool vecWrapContains(Rcpp::XPtr<VecWrap> w, double val) {
return w->contains(val);
}
v <- 1:10e7
w <- createVecWrap(v)
vecWrapContains(w, 10000)
# [1] TRUE
rm(v)
gc()
# used (Mb) gc trigger (Mb) max used (Mb)
# Ncells 366583 19.6 750400 40.1 460000 24.6
# Vcells 100559876 767.3 145208685 1107.9 100560540 767.3
vecWrapContains(w, 10000)
# [1] TRUE
Unrelated: consider using { } for your control flow structures, and don't get carried away with this->; both of those will improve the readability of your code IMO.

Related

C way to detect and 'separate' a binary expression

I have a bottleneck in my code in expressions like any(x >= b | x == y) for a large x.
I'd like to avoid the allocation x >= b | x == y. I've found that it's easy to write a function for particular cases.
SEXP eval_any_or2(SEXP x, SEXP b, SEXP y) {
R_xlen_t N = xlength(x);
if (xlength(y) != N || xlength(b) != 1) {
error("Wrong lengths.");
}
const int *xp = INTEGER(x);
const int *yp = INTEGER(y);
const int *bp = INTEGER(b);
bool o = false;
for (R_xlen_t i = 0; i < N; ++i) {
if (xp[i] >= bp[0] || xp[i] == yp[i]) {
o = true;
break;
}
}
SEXP ans = PROTECT(allocVector(LGLSXP, 1));
LOGICAL(ans)[0] = o ? TRUE : FALSE;
UNPROTECT(1);
return ans;
}
However, for clarity I'd like to keep as much of the natural syntax as possible, like any_or(x >= b, x == y). So I'd like to be able to detect whether a call is of the form <vector> <operator> <vector> when <operator> is one of the standard binary operators, and each <vector> is of equal length vectors length 1. Something like this:
any_or2 <- function(expr1, expr2) {
sexp1 <- substitute(expr1)
sexp2 <- substitute(expr2)
if (!is_binary_sexp(sexp1) || !is_binary_sexp(sexp2) {
# fall through to just basic R
return(any(expr1 | expr2))
}
# In C
eval_any_or2(...) # either the substituted expression or x,y,b
}
I've attempted the following C function which detects whether a substituted expression/call is a binary expression, but (a) I'm having trouble detecting whether the operator is a binary operator and (b) getting the vectors from the expression (x, y, b in the example) to use later (either in the same C function or as passed to a C function like the one above).
#define return_false SEXP ans = PROTECT(allocVector(LGLSXP, 1)); \
LOGICAL(ans)[0] = FALSE; \
UNPROTECT(1); \
return ans; \
SEXP is_binary_sexp(SEXP sx) {
if (TYPEOF(sx) != LANGSXP) {
return_false
}
// does it have three elements?
int len = 0;
SEXP el, nxt;
for (nxt = sx; nxt != R_NilValue || len > 4; el = CAR(nxt), nxt = CDR(nxt)) {
len++;
}
if (len != 3) {
return_false;
}
if (TYPEOF(CAR(sx)) != SYMSXP) {
return_false;
}
SEXP ans = PROTECT(allocVector(LGLSXP, 1));
LOGICAL(ans)[0] = TRUE;
UNPROTECT(1);
return ans;
}
In R I would write something like:
is_binary_sexp_R <- function(sexprA) {
# sexprA is the result of substitute()
is.call(sexprA) &&
length(sexprA) == 3L &&
match(as.character(sexprA[[1]]), c("!=", "==", "<=", ">=", "<", ">"), nomatch = 0L) &&
is.name(lhs <- sexprA[[2L]])
}
but I'd like to do as much as possible in C.

Can an Rcpp::IntegerVector (ever) be used with OpenMP?

I am probably being greedy for performance, but I've observed significant performance gains when combining Rcpp and OpenMP in possibly illict ways. I understand that "Calling any of the R API from threaded code is ‘for experts only’ and strongly discouraged." but I'm don't fully understand when C++ code may be implicitly doing this on Rcpp vectors. I understand RcppParallel has the RVector class but I understand this may involve taking a copy of the vector, which may wash away any performance gains.
I note the Rcpp gallery has (https://gallery.rcpp.org/articles/gerber-statistic/) which appears to access a NumericMatrix HIST_RETURN_RAW inside a parallel loop, so it seems "some" access is allowed, but I know/believe that some wrappers (like Rcpp::List) would not be permitted. Is atomicity the distinguishing characteristic?
Concretely, are any of the following acceptable uses of OpenMP? (i.e. are they all threadsafe, compliant with R's memory management, and free from undefined behaviour?)
#include <Rcpp.h>
#ifdef _OPENMP
#include <omp.h>
#endif
using namespace Rcpp;
// 1. No 'outside' R vector, but IntegerVector created outside omp region
// [[Rcpp::export]]
IntegerVector fastInitalize(int n, int nThread = 1) {
IntegerVector out = no_init(n);
#pragma omp parallel for num_threads(nThread)
for (int i = 0; i < n; ++i) {
out[i] = 0;
}
return out;
}
// 2. Simple access
// [[Rcpp::export]]
IntegerVector AddOMP(IntegerVector x, IntegerVector y, int nThread = 1) {
R_xlen_t N = x.length();
if (N != y.length()) {
stop("Lengths differ");
}
IntegerVector out = no_init(N);
#pragma omp parallel for num_threads(nThread)
for (R_xlen_t i = 0; i < N; ++i) {
out[i] = x[i] + y[i];
}
return out;
}
// 3. Access, copy inside Rcpp
// [[Rcpp::export]]
IntegerVector pmax0xy(IntegerVector x, IntegerVector y, int nThread = 1) {
R_xlen_t N = x.length();
if (N != y.length()) {
stop("Lengths differ");
}
IntegerVector out = clone(y);
#pragma omp parallel for num_threads(nThread)
for (R_xlen_t i = 0; i < N; ++i) {
if (x[i] > 0) {
out[i] = 0;
}
}
return out;
}
// 4. Access with omp + reduction
// [[Rcpp::export]]
int firstNonzero(IntegerVector x, int nThread = 1) {
R_xlen_t N = x.length();
int out = N;
#pragma omp parallel for num_threads(nThread) reduction(min : out)
for (R_xlen_t i = 0; i < N; ++i) {
if (x[i] != 0) {
out = (i < out) ? i : out;
}
}
return out;
}
// 5. Access with omp array reduction
// [[Rcpp::export]]
IntegerVector count_one_to_ten(IntegerVector x, int nThread = 1) {
R_xlen_t N = x.length();
if (N >= INT_MAX) {
stop("Possibly too many numbers.");
}
const int TEN = 10;
int numbers[TEN] = {}; // what if 10 was large?
#if defined _OPENMP && _OPENMP >= 201511
#pragma omp parallel for num_threads(nThread) reduction(+:numbers[:TEN])
#endif
for (R_xlen_t i = 0; i < N; ++i) {
int xi = x[i];
if (xi < 1 || xi > 10) {
continue;
}
numbers[xi - 1] += 1;
}
IntegerVector out(TEN);
for (int n = 0; n < TEN; ++n) {
out[n] = numbers[n];
}
return out;
}
// You can include R code blocks in C++ files processed with sourceCpp
// (useful for testing and development). The R code will be automatically
// run after the compilation.
//
/*** R
x <- sample(1:1200, size = 1e6, replace = TRUE)
y <- sample(1:1200, size = 1e6, replace = TRUE)
fastInitalize(1e6)[1]
head(AddOMP(x, y))
head(AddOMP(x, y, 2))
head(pmax0xy(x, y))
head(pmax0xy(x, y, 2))
firstNonzero(x)
firstNonzero(x, 2)
count_one_to_ten(x, 2)
*/

Rcpp template class for custom input and output

I'm trying to create C++ function which will apply any funtion on R vector of any type. I've been reading and searching for an answer but my knowledge is still too chaotic and I can't put everything together. I was inspired by sapply example and some from Rcpp gallery but it's to advanced so far.
What I've already done is simple class which kinda-works, but I'm having problem even with this one. Error happens when I'm trying to call function which returns something else than numeric. However I don't know how to extend function to work with custom output type.
At this point, I don't know how:
Obtain type of R Function return value and use this type do define out - same size as x
Alternatively - use std::string type argument in apply_fun which could switch OUTTYPE
Pass any x to the class and use in f - I think I've managed this correctly with <int XTYPE>
Perhaps answer to this question might be to complex so I appreciate all hints. Below I present current progress. Thanks!
Rcpp class
#include <Rcpp.h>
namespace apply {
template <int XTYPE>
class SomeClass {
private:
Rcpp::Vector<XTYPE> x;
Rcpp::Function f;
public:
Rcpp::Vector<XTYPE> run() {
typedef typename Rcpp::traits::storage_type<XTYPE>::type STORAGE;
int n = x.size();
Rcpp::Vector<XTYPE> out(n);
for (unsigned int i{0}; i < n; i++) {
out(i) = Rcpp::as<STORAGE>(f(x(i)));
// Rcpp::Rcout << out(i) << std::endl;
}
return out;
}
SomeClass (Rcpp::Vector<XTYPE> x, Rcpp::Function f)
: x{x}, f{f} {
std::cout << "Initialized SomeClass" << std::endl;
}
};
}
Exported Rcpp function
//' #export
//[[Rcpp::export]]
Rcpp::RObject apply_fun(Rcpp::RObject x,
Rcpp::Function f) {
if (TYPEOF(x) == INTSXP) {
apply::SomeClass<13> r{Rcpp::as<Rcpp::IntegerVector>(x), f};
return r.run();
} else if (TYPEOF(x) == REALSXP) {
apply::SomeClass<14> r{Rcpp::as<Rcpp::NumericVector>(x), f};
return r.run();
} else if (TYPEOF(x) == STRSXP) {
apply::SomeClass<16> r{Rcpp::as<Rcpp::CharacterVector>(x), f};
return r.run();
} else if (TYPEOF(x) == LGLSXP) {
apply::SomeClass<10> r{Rcpp::as<Rcpp::LogicalVector>(x), f};
return r.run();
} else if (TYPEOF(x) == CPLXSXP) {
apply::SomeClass<15> r{Rcpp::as<Rcpp::ComplexVector>(x), f};
return r.run();
} else {
Rcpp::stop("Invalid data type - only integer, numeric, character, factor, date, logical, complex vectors are possible.");
}
return R_NilValue;
}
R calls
apply_fun(c(1.5, 2.5, 3.5), f = function(x) { x + 10})
# 11.5 12.5 13.5
apply_fun(letters[1:3], f = function(x) paste(x, "-"))
# Error in apply_run(letters[1:3], f = function(x) x) :
# Evaluation error: unimplemented type 'char' in 'eval'

RcppParallel RVector push_back or something similar?

I am using RcppParallel to speed up some calculations. However, I am running out of memory in the process, so I would like to save results within the Parallel loop that are pass some relevance threshold. Below is a toy example to illustrate my point:
#include <Rcpp.h>
#include <RcppParallel.h>
using namespace Rcpp;
// [[Rcpp::depends(RcppParallel)]]
// [[Rcpp::plugins(cpp11)]]
struct Example : public RcppParallel::Worker {
RcppParallel::RVector<double> xvals, xvals_output, yvals;
Example(const NumericVector & xvals, NumericVector & yvals, NumericVector & xvals_output) :
xvals(xvals), xvals_output(xvals_output), yvals(yvals) {}
void operator()(std::size_t begin, size_t end) {
for(std::size_t i=begin; i < end; i++) {
double y = xvals[i] * (xvals[i] - 1);
// if(y < 0) {
// xvals_output.push_back(xvals[i]);
// yvals.push_back(y);
// }
xvals_output[i] = xvals[i];
yvals[i] = y;
}
}
};
// [[Rcpp::export]]
List find_values(NumericVector xvals) {
NumericVector xvals_output(xvals.size());
NumericVector yvals(xvals.size());
Example ex(xvals, yvals, xvals_output);
parallelFor(0, xvals.size(), ex);
List L = List::create(xvals_output, yvals);
return(L);
}
The R code would be:
find_values(seq(-10,10, by=0.5))
The commented out code is what I would like to do.
That is, I would like to initialize an empty vector, and append only the y-values that pass a certain threshold and also the associated x-values.
In my real usage, I am calculating a MxN matrix, so memory is an issue.
What is the correct way to approach this issue?
If anyone ever comes across a similar problem, here's a solution using "concurrent_vector" from TBB (which RcppParallel uses under the hood and is available as a header).
#include <Rcpp.h>
#include <RcppParallel.h>
#include <tbb/concurrent_vector.h>
using namespace Rcpp;
// [[Rcpp::depends(RcppParallel)]]
// [[Rcpp::plugins(cpp11)]]
struct Example : public RcppParallel::Worker {
RcppParallel::RVector<double> xvals;
tbb::concurrent_vector< std::pair<double, double> > &output;
Example(const NumericVector & xvals, tbb::concurrent_vector< std::pair<double, double> > &output) :
xvals(xvals), output(output) {}
void operator()(std::size_t begin, size_t end) {
for(std::size_t i=begin; i < end; i++) {
double y = xvals[i] * (xvals[i] - 1);
if(y < 0) {
output.push_back( std::pair<double, double>(xvals[i], y) );
}
}
}
};
// [[Rcpp::export]]
List find_values(NumericVector xvals) {
tbb::concurrent_vector< std::pair<double, double> > output;
Example ex(xvals,output);
parallelFor(0, xvals.size(), ex);
NumericVector xout(output.size());
NumericVector yout(output.size());
for(int i=0; i<output.size(); i++) {
xout[i] = output[i].first;
yout[i] = output[i].second;
}
List L = List::create(xout, yout);
return(L);
}
Output:
> find_values(seq(-10,10, by=0.5))
[[1]]
[1] 0.5
[[2]]
[1] -0.25

When calling same Rcpp function several times different results are returned

I have written parallel implementation of sums in groups using RcppParallel.
// [[Rcpp::depends(RcppParallel)]]
#include <Rcpp.h>
#include <RcppParallel.h>
using namespace Rcpp;
using namespace RcppParallel;
struct SumsG: public Worker
{
const RVector<double> v;
const RVector<int> gi;
RVector<double> sg;
SumsG(const NumericVector v, const IntegerVector gi, NumericVector sg): v(v), gi(gi), sg(sg) {}
SumsG(const SumsG& p, Split): v(p.v), gi(p.gi), sg(p.sg) {}
void operator()(std::size_t begin, std::size_t end) {
for (std::size_t i = begin; i < end; i++) {
sg[gi[i]] += v[i];
}
}
void join(const SumsG& p) {
for(std::size_t i = 0; i < sg.length(); i++) {
sg[i] += p.sg[i];
}
}
};
// [[Rcpp::export]]
List sumsingroups(NumericVector v, IntegerVector gi, int ni) {
NumericVector sg(ni);
SumsG p(v, gi, sg);
parallelReduce(0, v.length(), p);
return List::create(_["sg"] = p.sg);
}
It compiles using Rcpp::sourceCpp. Now when I call it from R sumsingroups(1:10, rep(0:1, each = 5), 2) several times I get the right answer (15 40) and then something different (usually some multiplicative of the right answer). Running
res <- sumsingroups(1:10, rep(0:1, each = 5), 2)
for(i in 1:1000) {
tmp <- sumsingroups(1:10, rep(0:1, each = 5), 2)
if(res[[1]][1] != tmp[[1]][1]) break
Sys.sleep(0.1)
}
breaks at random iteration returning
$sg
[1] 60 160
or
$sg
[1] 30 80
I am new to Rcpp and RcppParallel and do not know what could cause such behavior.
Update. Things that did not help:
Added for (std::size_t i = 0; i < sg.length(); i++) sg[i] = 0; to
both of constructors.
Changed names so that they are different in
Worker definition and in function implementation.
Try this.
#include <Rcpp.h>
using namespace Rcpp;
// [[Rcpp::depends(RcppParallel)]]
#include <RcppParallel.h>
using namespace RcppParallel;
struct SumsInGroups5: public Worker
{
const RVector<double> v;
const RVector<int> g;
std::vector<double> s;
SumsInGroups5(const NumericVector v, const IntegerVector g): v(v), g(g), s(*std::max_element(g.begin(), g.end()) + 1, 0.0){ }
SumsInGroups5(const SumsInGroups5& p, Split): v(p.v), g(p.g), s(*std::max_element(g.begin(), g.end()) + 1, 0.0) {}
void operator()(std::size_t begin, std::size_t end) {
for (std::size_t i = begin; i < end; ++i) {
s[g[i]]+=v[i];
}
}
void join(const SumsInGroups5& rhs) {
for(std::size_t i = 0; i < s.size(); i++) {
s[i] += rhs.s[i];
}
}
};
// [[Rcpp::export]]
NumericVector sg5(NumericVector v, IntegerVector g) {
SumsInGroups5 p(v, g);
parallelReduce(0, v.length(), p);
return wrap(p.s);
}
/*** R
a <- 1:10
g <- c(rep(0,5),rep(1,5))
bb <- lapply(1:10000,function(x)sg5(a,g))
cc<-do.call("rbind",bb)
unique(cc)
*/
Compared to my other tries this code did not produce weird result in the same cases other code did. Not very assuring.

Resources