Visualize blinks in conversation - r

Question

I have conversational data with pupillary data showing when ppl blink, like so (reproducible data below):
df
# A tibble: 6 x 8
# Groups: Blinks_grp [6]
Speaker Utterance Starttime_ms Endtime_ms Blink_onset Blink_offset Blinks_grp Blink_dur
<chr> <chr> <int> <dbl> <dbl> <dbl> <dbl> <dbl>
1 ID16.B an Americ… 289569 293940 289879 289946 113 67
2 ID16.B an Americ… 289569 293940 290696 290879 114 183
3 ID16.B an Americ… 289569 293940 290962 291046 115 84
4 ID16.A [°gotcha°] 290604 291004 290696 290879 116 183
5 ID16.B =↓my fath… 300938 302140 301529 301612 117 83
6 ID16.B =↓my fath… 300938 302140 302062 302146 118 78
I want to visualize when blinks occur relative to speech (in column Utterance). So far I've come up with this code:
df %>%
mutate(Utterance = paste0(sub(".*(.)$", "\\1",Speaker), ": ", Utterance),
Utterance = factor(Utterance, levels = unique(Utterance))) %>%
ggplot(aes(x = Blink_onset, xend = Blink_offset,
y = Blinks_grp, yend = Blinks_grp)) +
geom_segment(size = 3) +
facet_wrap(~ Utterance, ncol = 1, scales= "free_x")
which produces this graph:
However, the graph does not make the relationship Utterance v. blinks clear enough:
it fails to take into account the duration of the Utterance
it fails to indicate whether blinks occur (partially) before the start or after the end of Utterances
if more than one blink occurs during one Utterance it puts these blinks on different lines
So what I'm looking for is a visulization that shows both blinks and Utterance and makes clear where blinks occur relative to Utterance. What I have in mind looks somewhat like this:
Reproducible data:
structure(list(Speaker = c("ID16.B", "ID16.B", "ID16.B", "ID16.A",
"ID16.B", "ID16.B"), Utterance = c("an American family that (.) [uh] moved to Germany in <nineteen ninety one>",
"an American family that (.) [uh] moved to Germany in <nineteen ninety one>",
"an American family that (.) [uh] moved to Germany in <nineteen ninety one>",
"[°gotcha°]", "=↓my father's↓ like ~°we're going to Germany°~",
"=↓my father's↓ like ~°we're going to Germany°~"), Starttime_ms = c(289569L,
289569L, 289569L, 290604L, 300938L, 300938L), Endtime_ms = c(293940,
293940, 293940, 291004, 302140, 302140), Blink_onset = c(289879,
290696, 290962, 290696, 301529, 302062), Blink_offset = c(289946,
290879, 291046, 290879, 301612, 302146), Blinks_grp = c(113,
114, 115, 116, 117, 118), Blink_dur = c(67, 183, 84, 183, 83,
78)), row.names = c(NA, -6L), groups = structure(list(Blinks_grp = c(113,
114, 115, 116, 117, 118), .rows = structure(list(1L, 2L, 3L,
4L, 5L, 6L), ptype = integer(0), class = c("vctrs_list_of",
"vctrs_vctr", "list"))), row.names = c(NA, -6L), class = c("tbl_df",
"tbl", "data.frame"), .drop = TRUE), class = c("grouped_df",
"tbl_df", "tbl", "data.frame"))

Answer 1

The main problem here is, how to shape the data so that ggplot can plot it. Here's a proposal: in the essence I'm replacing blink_onset and blink_offset with an attribute that tells whehter an entry in the table is blinking or talking, then ggplot can easily draw for each event a separate line:
## Preparing the data:
df <- structure(list(Speaker = c("ID16.B", "ID16.B", "ID16.B", "ID16.A",
"ID16.B", "ID16.B"), Utterance = c("an American family that (.) [uh] moved to Germany in <nineteen ninety one>",
"an American family that (.) [uh] moved to Germany in <nineteen ninety one>",
"an American family that (.) [uh] moved to Germany in <nineteen ninety one>",
"[°gotcha°]", "=↓my father's↓ like ~°we're going to Germany°~",
"=↓my father's↓ like ~°we're going to Germany°~"), Starttime_ms = c(289569L,
289569L, 289569L, 290604L, 300938L, 300938L), Endtime_ms = c(293940,
293940, 293940, 291004, 302140, 302140), Blink_onset = c(289879,
290696, 290962, 290696, 301529, 302062), Blink_offset = c(289946,
290879, 291046, 290879, 301612, 302146), Blinks_grp = c(113,
114, 115, 116, 117, 118), Blink_dur = c(67, 183, 84, 183, 83,
78)), row.names = c(NA, -6L), groups = structure(list(Blinks_grp = c(113,
114, 115, 116, 117, 118), .rows = structure(list(1L, 2L, 3L,
4L, 5L, 6L), ptype = integer(0), class = c("vctrs_list_of",
"vctrs_vctr", "list"))), row.names = c(NA, -6L), class = c("tbl_df",
"tbl", "data.frame"), .drop = TRUE), class = c("grouped_df",
"tbl_df", "tbl", "data.frame"))
df <- df %>%
mutate(Utterance = paste0(sub(".*(.)$", "\\1",Speaker), ": ", Utterance),
Utterance = factor(Utterance, levels = unique(Utterance)))
## separate into a "Blink" and a "talk" data frame, add an attribute "Event" that represents talking and blinking:
blink_df <- df %>% select(Speaker, Utterance, Blink_onset, Blink_offset, Blinks_grp) %>%
mutate(Starttime_ms = Blink_onset, Endtime_ms = Blink_offset, Event = "blink")
talk_df <- df %>% select(Speaker, Utterance, Starttime_ms, Endtime_ms, Blinks_grp) %>%
mutate(Event = "talk")
## combine datasets again:
plot_df <- bind_rows(talk_df, blink_df)
## and plot, using "Event" as attribute to separate talking and blinking lines:
plot_df %>%
ggplot(aes(x = Starttime_ms, xend = Endtime_ms,
y = Event, yend = Event, colour = Event)) +
geom_segment(size = 3) +
facet_wrap(~ Utterance, ncol = 1, scales= "free_x")