How can I visualize 1D numeric data with R / tikz? - r

Question

I have a text file with 166898 lines where each line has a single non-negative number. I would like to visualize it in the following way:
The x-axis should range from minimum_in_file to maximum_in_file.
The y-axis should range from 1 to 166898
The graph / bar plot should be increasing and visualize how many numbers are equal to the x-value or lower.
Create data
#!/usr/bin/env python
import random
minimum_in_file = 0
maximum_in_file = 378864471
numbers = []
for i in range(166898):
numbers.append(random.randint(minimum_in_file, maximum_in_file))
numbers = sorted(numbers)
with open("times-sorted.txt", 'a') as f:
for number in numbers:
f.write(str(number) + "\n")
Real data
When I execute dput(head(mydata,20)) I get:
structure(list(X0 = c(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L)), .Names = "X0", row.names = c(NA,
20L), class = "data.frame")
and
> dput(head(mydata,1000))
structure(list(X0 = c(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
0L, 0L, 0L, 0L, 0L, 0L, 1L, 2L, 2L, 3L, 3L, 5L, 6L, 7L, 8L, 8L,
8L, 8L, 11L, 12L, 13L, 13L, 13L, 16L, 18L, 19L, 20L, 20L, 23L,
25L, 26L, 27L, 35L, 37L, 37L, 39L, 41L, 41L, 45L, 46L, 47L, 48L,
48L, 48L, 50L, 52L, 53L, 55L, 56L, 62L, 65L, 66L, 67L, 67L, 70L,
79L, 79L, 80L, 83L, 85L, 86L, 88L, 88L, 89L, 91L, 96L, 97L, 99L,
100L, 101L, 101L, 101L, 102L, 103L, 104L, 104L, 107L, 109L, 109L,
109L, 109L, 111L, 111L, 111L, 111L, 112L, 112L, 112L, 112L, 113L,
113L, 114L, 114L, 114L, 115L, 115L, 115L, 116L, 117L, 118L, 119L,
120L, 120L, 124L, 124L, 124L, 124L, 125L, 125L, 127L, 127L, 128L,
128L, 128L, 129L, 129L, 129L, 130L, 130L, 131L, 132L, 132L, 132L,
133L, 133L, 134L, 134L, 134L, 134L, 136L, 136L, 137L, 137L, 138L,
139L, 140L, 141L, 141L, 142L, 143L, 143L, 143L, 144L, 144L, 144L,
145L, 145L, 146L, 147L, 147L, 149L, 149L, 150L, 150L, 150L, 150L,
150L, 151L, 151L, 151L, 151L, 151L, 152L, 152L, 153L, 154L, 154L,
154L, 154L, 155L, 156L, 157L, 157L, 158L, 158L, 158L, 158L, 159L,
160L, 160L, 160L, 160L, 161L, 161L, 163L, 163L, 163L, 164L, 164L,
164L, 164L, 164L, 165L, 165L, 166L, 166L, 167L, 167L, 167L, 167L,
168L, 168L, 168L, 169L, 169L, 170L, 170L, 171L, 171L, 172L, 172L,
172L, 172L, 173L, 173L, 173L, 174L, 174L, 175L, 175L, 175L, 176L,
176L, 176L, 176L, 177L, 177L, 177L, 177L, 177L, 179L, 179L, 179L,
180L, 180L, 180L, 180L, 181L, 181L, 182L, 182L, 182L, 182L, 183L,
183L, 184L, 184L, 184L, 184L, 185L, 185L, 185L, 186L, 187L, 187L,
187L, 187L, 188L, 188L, 188L, 188L, 189L, 189L, 189L, 189L, 190L,
190L, 190L, 190L, 191L, 191L, 191L, 191L, 191L, 191L, 191L, 191L,
191L, 192L, 192L, 193L, 193L, 194L, 194L, 195L, 195L, 195L, 197L,
197L, 197L, 197L, 197L, 198L, 198L, 198L, 198L, 198L, 198L, 199L,
199L, 199L, 199L, 199L, 199L, 199L, 200L, 200L, 200L, 200L, 200L,
200L, 200L, 200L, 200L, 201L, 201L, 203L, 203L, 203L, 204L, 204L,
204L, 205L, 205L, 206L, 206L, 206L, 206L, 206L, 206L, 206L, 207L,
207L, 207L, 207L, 207L, 207L, 208L, 208L, 209L, 209L, 209L, 209L,
209L, 209L, 210L, 210L, 210L, 210L, 210L, 211L, 211L, 212L, 213L,
213L, 213L, 213L, 215L, 215L, 215L, 215L, 215L, 215L, 215L, 215L,
215L)), .Names = "X0", row.names = c(NA, 1000L), class = "data.frame")
What I've tried
I sorted the data with the unix tool sort and tried the following with R version 3.0.1 (2013-05-16):
> mydata = read.csv("times-sorted.txt")
> accumulated_sum<-cumsum(mydata)
Warning message:
In lapply(X = x, FUN = .Generic, ...) :
integer overflow in 'cumsum'; use 'cumsum(as.numeric(.))'
> plot(accumulated_sum)
But I don't understand how to fix the warning and the result is crappy:
How can I visualize it in a nice way?

Answer 1

See this source code here (https://svn.r-project.org/R/trunk/src/main/cum.c), and the statement
if(sum > INT_MAX || sum < 1 + INT_MIN) where INT_MAX is .Machine$integer.max ,possibly this limit is being exceeded since you are applying cumsum to the entire dataset and not the variable of interest.
Since you have not posted your dataset structure, I think the row indices are being passed to cumsum and hence the warning,
N=166898
vec=1:N
#produces warning "Warning message: integer overflow in 'cumsum'; use 'cumsum(as.numeric(.))'"
cumsum(vec)
You need a cumulative frequency plot. Following is sample example adapted from
(http://www.r-tutor.com/elementary-statistics/quantitative-data/cumulative-frequency-graph)
Sample Example
#For reproducibility
set.seed(100)
N=166898
vec=1:N
#Assuming min_val, max_val
min_val = 0
max_val = 378864471
min_break = 1e8
max_break = 4e8
seq_by = 1e8
#Create random values dataset
random_values = sample(min_val:max_val,N,replace = T)
DF=data.frame(vec,random_values)
#Compute Cumulative Frequency
#You can control the buckets by appropriate inputs to breaks
breaks = seq(min_break, max_break, by=seq_by)
#Creates buckets [x,y), [y,z) etc.
DF.cut = cut( DF$random_values, breaks, right=FALSE)
#Computes count of observations in various buckets and cumulative frequency
DF.freq=table(DF.cut)
DF.cumfreq = c(0, cumsum(DF.freq))
#Plot Data
plot(breaks, DF.cumfreq,main="Cumulative Frequency of XYZ",xlab="Range of Values",ylab="# of Observations < X")
lines(breaks, DF.cumfreq)
Your data
I have plotted the below plot using the data sample you provided, but the following should work for your file(s) now.
#Replace the appropriate filename here
mydata = read.table("times-sorted.txt")
min_val_new = min(mydata)
max_val_new = max(mydata)
breaks_new = seq(from=min_val_new,to=max_val_new,length.out=5)
#Creates buckets [x,y), [y,z) etc.
DF.cut_new = cut(mydata[,1], breaks_new, right=FALSE)
#Computes count of observations in various buckets and cumulative frequency
DF.freq_new=table(DF.cut_new)
DF.cumfreq_new = c(0, cumsum(DF.freq_new))
#Plot Data
plot(breaks_new, DF.cumfreq_new,main="Cumulative Frequency of ABC",xlab="Range of Values",ylab="# of Observations < X")
lines(breaks_new, DF.cumfreq_new)
Exponential Plot
Define your breakpoints cutoff1=10000 and cutoff2=60000 and include them in 'breaks' calculation and plotting using ggplot2 with log axis
set.seed(100)
require(ggplot2)
N=166898
vec=1:N
#Assuming min_val, max_val
min_val = 0
max_val = 378864471
min_break=0
max_break=4e8
#Create random values dataset
random_values = sample(min_val:max_val,N,replace = T)
DF=data.frame(vec,random_values)
#Define your data breakpoints
cutoff1=10000
cutoff2=60000
#Compute Cumulative Frequency
#You can control the buckets by appropriate inputs to breaks
breaks = c(min_break,cutoff1,seq(cutoff2, max_break, length.out=30))
#Creates buckets [x,y), [y,z) etc.
DF.cut = cut( DF$random_values, breaks, right=FALSE)
#Computes count of observations in various buckets and cumulative frequency
DF.freq=table(DF.cut)
DF.cumfreq = c(0, cumsum(DF.freq))
#Plot Data
#plot(breaks, DF.cumfreq,main="Cumulative Frequency of XYZ",xlab="Range of Values",ylab="# of Observations < X")
#lines(breaks, DF.cumfreq)
gg.df=data.frame(breaks,DF.cumfreq)
ggplot(gg.df, aes(x = breaks,y=DF.cumfreq)) + geom_line() + scale_x_log10() +
xlab("Range of Values: Log Axis") +
ylab("# of Observations < X") +
ggtitle("Cumulative Frequency of Variable")