Reading England and Wales Charity Commission bcp files in R - r

Question

I'm trying to read .bcp files provided by https://register-of-charities.charitycommission.gov.uk/register/full-register-download in R. I have been trying previously answered questions here, but readChar does not seem to read everything in all files, namely it breaks for extract_charity.bcp.
So I have thought of readBin and tried to read extract_charity.bcp like this:
library(stringr)
b <- readBin("extract_charity.bcp", "character", n = 300000, size = NA_integer_,
endian = .Platform$endian)
c<- paste0(b, collapse = "" ) #put it back as one large character string
d<- str_locate_all(c, "\\*\\#\\#\\*\\d") #find row breaks followed by a digit
e <- d[[1]]
flags <- e[,1]
f <- c()
f[1] <- substr(c, 1, flags[1]-1)
for (i in 2:length(flags)) f[i]<- substr(c, flags[i-1]+4, flags[i]-1) #removes row breaks
export <- matrix(nrow = 372432, ncol = 18)
exportF <- matrix(nrow = 0, ncol = 18)
for (j in 1:length(flags)) {
new_row <- str_split( f[j], "\\#\\*\\*\\#" )[[1]] #removes column breaks
if (length(new_row)==18) { export[j, ] <- new_row #if correct number of columns
} else { print(j)
exportF <- rbind(exportF, new_row) }}
However, there are 49 errors - all of the same type. There is a strange character string inserted at various places across the table - currently it is "P`j[Ÿ " but when I run the script again, it is "°Tj[Ÿ ", so it provides different string every time I run the script, so I cannot run the script to remove it manually:
str_replace_all(c, problem, "")
Error in stri_replace_all_regex(string, pattern, fix_replacement(replacement), :
Missing closing bracket on a bracket expression. (U_REGEX_MISSING_CLOSE_BRACKET)

Answer 1

Just to let the world know, this can be done.
In first pass, the file is parsed and problems are stored in exportF, where it is identified and deleted from original parsing output. Then in the second pass, that is parsed correctly.
This is a mess, but it works, and pretty fast, too.
library(stringr)
library(stringi)
b <- readBin("extract_charity.bcp", "character", n = 300000, size = NA_integer_)
c<- paste0(b, collapse = "" )
tt<- str_locate_all(c, "\\*\\#\\#\\*\\d")
e <- tt[[1]]
flags <- e[,1]
f <- c()
f[1] <- substr(c, 1, flags[1]-1)
for (i in 2:length(flags)) {
f[i]<- substr(c, flags[i-1]+4, flags[i]-1)
}
export <- matrix(nrow = length(flags), ncol = 18)
exportF <- matrix(nrow = 0, ncol = 18)
for (j in 1:length(flags)) {
new_row <- str_split( f[j], "\\#\\*\\*\\#" )[[1]]
if (length(new_row)==18) { export[j, ] <- new_row
} else {print(flags[j])
exportF <- rbind(exportF, new_row) }}
#go trough the first line and see where the problem is and locate its position
problem <- str_sub(as.character(exportF[1,8]), 5, 10)
#CHECK TO SEE IF CORRECT
problem %in% str_sub(exportF[1,8], 5, 10)
problem %in% exportF[1,8]
str_detect(c,problem )
str_detect(b[324],problem )
#d <-stri_replace_all_charclass(b, problem, "")
str_detect(d,problem )
r<- gsub(problem, "", b )
str_detect(r,problem )
#now go again but with clean data
r<- paste0(r, collapse = "" )
tt<- str_locate_all(r, "\\*\\#\\#\\*\\d")
e <- tt[[1]]
flags <- e[,1]
f <- c()
f[1] <- substr(r, 1, flags[1]-1)
for (i in 2:length(flags)) {
f[i]<- substr(r, flags[i-1]+4, flags[i]-1)
}
#g<- str_split(f[372432], "\\#\\*\\*\\#")[[1]]
export <- matrix(nrow = 372434, ncol = 18)
exportF <- matrix(nrow = 0, ncol = 18)
for (j in 1:length(flags)) {
new_row <- str_split( f[j], "\\#\\*\\*\\#" )[[1]]
if (length(new_row)==18) { export[j, ] <- new_row
} else {print(flags[j])
exportF <- rbind(exportF, new_row) }}
write.csv(export, "extract_charity2021.csv", row.names = F)
leaving it here for either future myself or someone in need of doing this.