I have data from an automated telephone survey. This survey includes 2 call-backs when a caller gets disconnected.
Here is data from one respondent. I have thousands, though, with similar data that I need to process.
The column is the respondent's ID, the second is an ordering ID (because the timestamp isn't out to milliseconds), the third is the interaction between the survey tool and the respondent's phone (essentially the respondent), the fourth is the datetime stamp, and the final is the difference in time between rows that I calculated at an earlier point.
test <- data.frame(matrix(c(
111, 2340, 'Enqueueing call', '12/2/19 14:53:57', NA,
111, 6174, 'Call expired, will be retried in next schedule window', '12/2/19 20:14:22', 19225,
111, 10386, 'Answer', '12/3/19 15:48:56', 70474,
111, 10387, 'Contacted', '12/3/19 15:48:56', 0,
111, 10388, 'Intro', '12/3/19 15:48:56', 0,
111, 10389, 'Timeout', '12/3/19 15:49:16', 20,
111, 10390, 'Intro', '12/3/19 15:49:16', 0,
111, 10391, '1', '12/3/19 15:49:30', 14,
111, 10392, 'Started', '12/3/19 15:49:30', 0,
111, 10393, 'Q1', '12/3/19 15:49:30', 0,
111, 10394, '1', '12/3/19 15:49:45', 15,
111, 10395, 'Q2', '12/3/19 15:49:45', 0,
111, 10396, 'Timeout', '12/3/19 15:49:54', 9,
111, 10397, 'Q2', '12/3/19 15:49:54', 0,
111, 10398, 'Timeout', '12/3/19 15:50:03', 9,
111, 10399, 'Q2', '12/3/19 15:50:03', 0,
111, 10400, 'Timeout', '12/3/19 15:50:11', 8,
111, 17658, 'Timeout. Call failed.', '12/4/19 17:50:27', 93616,
111, 19932, 'Call expired, will be retried in next schedule window', '12/4/19 20:45:17', 10490,
111, 25647, 'Call expired, will be retried in next schedule window', '12/5/19 20:33:27', 85690,
111, 31516, 'Call expired, will be retried in next schedule window', '12/6/19 20:17:18', 85431,
111, 36781, 'Call expired, will be retried in next schedule window', '12/7/19 20:02:16', 85498,
111, 38718, 'Answer', '12/8/19 10:24:07', 51711,
111, 38719, 'Q2', '12/8/19 10:24:07', 0,
111, 38720, 'Timeout', '12/8/19 10:24:16', 9,
111, 38721, 'Q2', '12/8/19 10:24:16', 0,
111, 38722, 'Timeout', '12/8/19 10:24:24', 8,
111, 38723, 'Q2', '12/8/19 10:24:24', 0,
111, 38724, 'Timeout', '12/8/19 10:24:33', 9,
111, 45339, 'Timeout. Call failed.', '12/9/19 12:25:08', 93635,
111, 49026, 'Call expired, will be retried in next schedule window', '12/9/19 20:31:26', 29178,
111, 53972, 'Call expired, will be retried in next schedule window', '12/10/19 20:14:38', 85392,
111, 58277, 'Answer', '12/11/19 17:55:29', 78051,
111, 58278, 'Q2', '12/11/19 17:55:29', 0,
111, 58279, 'Timeout', '12/11/19 17:55:38', 9,
111, 58280, 'Q2', '12/11/19 17:55:38', 0,
111, 58281, 'Timeout', '12/11/19 17:55:46', 8,
111, 58282, 'Q2', '12/11/19 17:55:46', 0,
111, 58283, '61', '12/11/19 17:55:57', 11,
111, 58284, 'Q3', '12/11/19 17:55:57', 0,
111, 58285, '4', '12/11/19 17:56:13', 16,
111, 58286, 'Q4', '12/11/19 17:56:13', 0,
111, 58288, '3', '12/11/19 17:56:42', 29,
111, 58289, 'Interim partial', '12/11/19 17:56:42', 0,
111, 58290, 'Q5', '12/11/19 17:56:42', 0,
111, 58291, '3', '12/11/19 17:56:59', 17,
111, 58292, 'Q6', '12/11/19 17:56:59', 0,
111, 58293, '3', '12/11/19 17:57:25', 26,
111, 58294, 'Q7', '12/11/19 17:57:25', 0,
111, 58295, '3', '12/11/19 17:57:38', 13,
111, 58296, 'Q8', '12/11/19 17:57:38', 0,
111, 58297, '3', '12/11/19 17:57:50', 12,
111, 58298, 'Q9', '12/11/19 17:57:50', 0,
111, 58299, 'Timeout', '12/11/19 17:58:09', 19,
111, 58300, 'Q9', '12/11/19 17:58:09', 0,
111, 58301, '10', '12/11/19 17:58:32', 23,
111, 58302, 'Q10', '12/11/19 17:58:32', 0,
111, 58303, '1', '12/11/19 17:58:49', 17,
111, 58304, 'Q11', '12/11/19 17:58:49', 0,
111, 58307, '3', '12/11/19 17:59:02', 13,
111, 58308, 'Q12', '12/11/19 17:59:02', 0,
111, 58309, 'Timeout', '12/11/19 17:59:23', 21,
111, 58310, 'Q13', '12/11/19 17:59:23', 0,
111, 58311, 'Timeout', '12/11/19 17:59:44', 21,
111, 58312, 'Q13', '12/11/19 17:59:44', 0,
111, 58313, '4', '12/11/19 17:59:51', 7,
111, 58314, 'Q14', '12/11/19 17:59:51', 0,
111, 58318, '2', '12/11/19 18:00:19', 28,
111, 58319, 'Q15', '12/11/19 18:00:19', 0,
111, 58320, '3', '12/11/19 18:00:36', 17,
111, 58321, 'Q16', '12/11/19 18:00:36', 0,
111, 58322, '4', '12/11/19 18:01:04', 28,
111, 58323, 'Q17', '12/11/19 18:01:04', 0,
111, 58324, '1', '12/11/19 18:01:18', 14,
111, 58325, 'Q18', '12/11/19 18:01:18', 0,
111, 58327, '4', '12/11/19 18:01:42', 24,
111, 58328, 'Q19', '12/11/19 18:01:42', 0,
111, 58329, '1', '12/11/19 18:01:56', 14,
111, 58330, 'Q20', '12/11/19 18:01:56', 0,
111, 58331, '1', '12/11/19 18:02:19', 23,
111, 58332, 'Q21', '12/11/19 18:02:19', 0,
111, 58333, '1', '12/11/19 18:02:28', 9,
111, 58334, 'Q22', '12/11/19 18:02:28', 0,
111, 58335, '2', '12/11/19 18:02:52', 24,
111, 58336, 'Completed', '12/11/19 18:02:52', 0,
111, 58337, 'Complete Message', '12/11/19 18:02:52', 0,
111, 58338, 'Thank you', '12/11/19 18:02:52', 0),
nrow=87, ncol=5, byrow=T,
dimnames=list(c(NULL), c("Respondent.ID", "order.ID", "Interaction", "Datetime", "difftime"))))
I need to sum up the time for difftime but only based on certain conditions. Essentially, I want the data to look like the below, so I can sum the time where Include = 1.
Condition 1: All the rows between the row after Interaction = "Answer" and the row before Interaction = "Timeout. Call failed." should be Include = 1.
Condition 2: All the rows between the row after Interaction = "Answer" and the row before Interaction = "Completed" should be Include = 1.
new <- data.frame(matrix(c(
111, 2340, 'Enqueueing call', ' 12/2/19 14:53:57 ', NA, 0,
111, 6174, 'Call expired, will be retried in next schedule window', ' 12/2/19 20:14:22 ', 19225, 0,
111, 10386, 'Answer', ' 12/3/19 15:48:56 ', 70474, 0,
111, 10387, 'Contacted', ' 12/3/19 15:48:56 ', 0, 1,
111, 10388, 'Intro', ' 12/3/19 15:48:56 ', 0, 1,
111, 10389, 'Timeout', ' 12/3/19 15:49:16 ', 20, 1,
111, 10390, 'Intro', ' 12/3/19 15:49:16 ', 0, 1,
111, 10391, '1', ' 12/3/19 15:49:30 ', 14, 1,
111, 10392, 'Started', ' 12/3/19 15:49:30 ', 0, 1,
111, 10393, 'Q1', ' 12/3/19 15:49:30 ', 0, 1,
111, 10394, '1', ' 12/3/19 15:49:45 ', 15, 1,
111, 10395, 'Q2', ' 12/3/19 15:49:45 ', 0, 1,
111, 10396, 'Timeout', ' 12/3/19 15:49:54 ', 9, 1,
111, 10397, 'Q2', ' 12/3/19 15:49:54 ', 0, 1,
111, 10398, 'Timeout', ' 12/3/19 15:50:03 ', 9, 1,
111, 10399, 'Q2', ' 12/3/19 15:50:03 ', 0, 1,
111, 10400, 'Timeout', ' 12/3/19 15:50:11 ', 8, 1,
111, 17658, 'Timeout. Call failed.', ' 12/4/19 17:50:27 ', 93616, 0,
111, 19932, 'Call expired, will be retried in next schedule window', ' 12/4/19 20:45:17 ', 10490, 0,
111, 25647, 'Call expired, will be retried in next schedule window', ' 12/5/19 20:33:27 ', 85690, 0,
111, 31516, 'Call expired, will be retried in next schedule window', ' 12/6/19 20:17:18 ', 85431, 0,
111, 36781, 'Call expired, will be retried in next schedule window', ' 12/7/19 20:02:16 ', 85498, 0,
111, 38718, 'Answer', ' 12/8/19 10:24:07 ', 51711, 0,
111, 38719, 'Q2', ' 12/8/19 10:24:07 ', 0, 1,
111, 38720, 'Timeout', ' 12/8/19 10:24:16 ', 9, 1,
111, 38721, 'Q2', ' 12/8/19 10:24:16 ', 0, 1,
111, 38722, 'Timeout', ' 12/8/19 10:24:24 ', 8, 1,
111, 38723, 'Q2', ' 12/8/19 10:24:24 ', 0, 1,
111, 38724, 'Timeout', ' 12/8/19 10:24:33 ', 9, 1,
111, 45339, 'Timeout. Call failed.', ' 12/9/19 12:25:08 ', 93635, 0,
111, 49026, 'Call expired, will be retried in next schedule window', ' 12/9/19 20:31:26 ', 29178, 0,
111, 53972, 'Call expired, will be retried in next schedule window', ' 12/10/19 20:14:38 ', 85392, 0,
111, 58277, 'Answer', ' 12/11/19 17:55:29 ', 78051, 0,
111, 58278, 'Q2', ' 12/11/19 17:55:29 ', 0, 1,
111, 58279, 'Timeout', ' 12/11/19 17:55:38 ', 9, 1,
111, 58280, 'Q2', ' 12/11/19 17:55:38 ', 0, 1,
111, 58281, 'Timeout', ' 12/11/19 17:55:46 ', 8, 1,
111, 58282, 'Q2', ' 12/11/19 17:55:46 ', 0, 1,
111, 58283, '61', ' 12/11/19 17:55:57 ', 11, 1,
111, 58284, 'Q3', ' 12/11/19 17:55:57 ', 0, 1,
111, 58285, '4', ' 12/11/19 17:56:13 ', 16, 1,
111, 58286, 'Q4', ' 12/11/19 17:56:13 ', 0, 1,
111, 58288, '3', ' 12/11/19 17:56:42 ', 29, 1,
111, 58289, 'Interim partial', ' 12/11/19 17:56:42 ', 0, 1,
111, 58290, 'Q5', ' 12/11/19 17:56:42 ', 0, 1,
111, 58291, '3', ' 12/11/19 17:56:59 ', 17, 1,
111, 58292, 'Q6', ' 12/11/19 17:56:59 ', 0, 1,
111, 58293, '3', ' 12/11/19 17:57:25 ', 26, 1,
111, 58294, 'Q7', ' 12/11/19 17:57:25 ', 0, 1,
111, 58295, '3', ' 12/11/19 17:57:38 ', 13, 1,
111, 58296, 'Q8', ' 12/11/19 17:57:38 ', 0, 1,
111, 58297, '3', ' 12/11/19 17:57:50 ', 12, 1,
111, 58298, 'Q9', ' 12/11/19 17:57:50 ', 0, 1,
111, 58299, 'Timeout', ' 12/11/19 17:58:09 ', 19, 1,
111, 58300, 'Q9', ' 12/11/19 17:58:09 ', 0, 1,
111, 58301, '10', ' 12/11/19 17:58:32 ', 23, 1,
111, 58302, 'Q10', ' 12/11/19 17:58:32 ', 0, 1,
111, 58303, '1', ' 12/11/19 17:58:49 ', 17, 1,
111, 58304, 'Q11', ' 12/11/19 17:58:49 ', 0, 1,
111, 58307, '3', ' 12/11/19 17:59:02 ', 13, 1,
111, 58308, 'Q12', ' 12/11/19 17:59:02 ', 0, 1,
111, 58309, 'Timeout', ' 12/11/19 17:59:23 ', 21, 1,
111, 58310, 'Q13', ' 12/11/19 17:59:23 ', 0, 1,
111, 58311, 'Timeout', ' 12/11/19 17:59:44 ', 21, 1,
111, 58312, 'Q13', ' 12/11/19 17:59:44 ', 0, 1,
111, 58313, '4', ' 12/11/19 17:59:51 ', 7, 1,
111, 58314, 'Q14', ' 12/11/19 17:59:51 ', 0, 1,
111, 58318, '2', ' 12/11/19 18:00:19 ', 28, 1,
111, 58319, 'Q15', ' 12/11/19 18:00:19 ', 0, 1,
111, 58320, '3', ' 12/11/19 18:00:36 ', 17, 1,
111, 58321, 'Q16', ' 12/11/19 18:00:36 ', 0, 1,
111, 58322, '4', ' 12/11/19 18:01:04 ', 28, 1,
111, 58323, 'Q17', ' 12/11/19 18:01:04 ', 0, 1,
111, 58324, '1', ' 12/11/19 18:01:18 ', 14, 1,
111, 58325, 'Q18', ' 12/11/19 18:01:18 ', 0, 1,
111, 58327, '4', ' 12/11/19 18:01:42 ', 24, 1,
111, 58328, 'Q19', ' 12/11/19 18:01:42 ', 0, 1,
111, 58329, '1', ' 12/11/19 18:01:56 ', 14, 1,
111, 58330, 'Q20', ' 12/11/19 18:01:56 ', 0, 1,
111, 58331, '1', ' 12/11/19 18:02:19 ', 23, 1,
111, 58332, 'Q21', ' 12/11/19 18:02:19 ', 0, 1,
111, 58333, '1', ' 12/11/19 18:02:28 ', 9, 1,
111, 58334, 'Q22', ' 12/11/19 18:02:28 ', 0, 1,
111, 58335, '2', ' 12/11/19 18:02:52 ', 24, 1,
111, 58336, 'Completed', ' 12/11/19 18:02:52 ', 0, 0,
111, 58337, 'Complete Message', ' 12/11/19 18:02:52 ', 0, 0,
111, 58338, 'Thank you', ' 12/11/19 18:02:52 ', 0, 0),
nrow=87, ncol=6, byrow=T,
dimnames=list(c(NULL), c("Respondent.ID", "order.ID", "Response", "Datetime", "difftime", "Include"))))
I tried adding columns indicating where the start and stop positions where but couldn't figure out how to index the rows between start and stop.
UPDATE
I found a case where we cut off the survey and end up cutting off some respondents. When that happens they don't get a Stop code in their Interaction data. As a result, the rle isn't picking up the final run length we want to capture. I've fiddled around with it but this is a new thing for me so I thought I'd post a version of the test dataset above that mimics the situation.
test2 <- data.frame(matrix(c(
111, 2340, 'Enqueueing call', '12/2/19 14:53:57', NA,
111, 6174, 'Call expired, will be retried in next schedule window', '12/2/19 20:14:22', 19225,
111, 10386, 'Answer', '12/3/19 15:48:56', 70474,
111, 10387, 'Contacted', '12/3/19 15:48:56', 0,
111, 10388, 'Intro', '12/3/19 15:48:56', 0,
111, 10389, 'Timeout', '12/3/19 15:49:16', 20,
111, 10390, 'Intro', '12/3/19 15:49:16', 0,
111, 10391, '1', '12/3/19 15:49:30', 14,
111, 10392, 'Started', '12/3/19 15:49:30', 0,
111, 10393, 'Q1', '12/3/19 15:49:30', 0,
111, 10394, '1', '12/3/19 15:49:45', 15,
111, 10395, 'Q2', '12/3/19 15:49:45', 0,
111, 10396, 'Timeout', '12/3/19 15:49:54', 9,
111, 10397, 'Q2', '12/3/19 15:49:54', 0,
111, 10398, 'Timeout', '12/3/19 15:50:03', 9,
111, 10399, 'Q2', '12/3/19 15:50:03', 0,
111, 10400, 'Timeout', '12/3/19 15:50:11', 8,
111, 17658, 'Timeout. Call failed.', '12/4/19 17:50:27', 93616,
111, 19932, 'Call expired, will be retried in next schedule window', '12/4/19 20:45:17', 10490,
111, 25647, 'Call expired, will be retried in next schedule window', '12/5/19 20:33:27', 85690,
111, 31516, 'Call expired, will be retried in next schedule window', '12/6/19 20:17:18', 85431,
111, 36781, 'Call expired, will be retried in next schedule window', '12/7/19 20:02:16', 85498,
111, 38718, 'Answer', '12/8/19 10:24:07', 51711,
111, 38719, 'Q2', '12/8/19 10:24:07', 0,
111, 38720, 'Timeout', '12/8/19 10:24:16', 9,
111, 38721, 'Q2', '12/8/19 10:24:16', 0,
111, 38722, 'Timeout', '12/8/19 10:24:24', 8,
111, 38723, 'Q2', '12/8/19 10:24:24', 0,
111, 38724, 'Timeout', '12/8/19 10:24:33', 9,
111, 45339, 'Timeout. Call failed.', '12/9/19 12:25:08', 93635,
111, 49026, 'Call expired, will be retried in next schedule window', '12/9/19 20:31:26', 29178,
111, 53972, 'Call expired, will be retried in next schedule window', '12/10/19 20:14:38', 85392,
111, 58277, 'Answer', '12/11/19 17:55:29', 78051,
111, 58278, 'Q2', '12/11/19 17:55:29', 0,
111, 58279, 'Timeout', '12/11/19 17:55:38', 9,
111, 58280, 'Q2', '12/11/19 17:55:38', 0,
111, 58281, 'Timeout', '12/11/19 17:55:46', 8,
111, 58282, 'Q2', '12/11/19 17:55:46', 0,
111, 58283, '61', '12/11/19 17:55:57', 11,
111, 58284, 'Q3', '12/11/19 17:55:57', 0,
111, 58285, '4', '12/11/19 17:56:13', 16,
111, 58286, 'Q4', '12/11/19 17:56:13', 0,
111, 58288, '3', '12/11/19 17:56:42', 29,
111, 58289, 'Interim partial', '12/11/19 17:56:42', 0,
111, 58290, 'Q5', '12/11/19 17:56:42', 0,
111, 58291, '3', '12/11/19 17:56:59', 17,
111, 58292, 'Q6', '12/11/19 17:56:59', 0,
111, 58293, '3', '12/11/19 17:57:25', 26,
111, 58294, 'Q7', '12/11/19 17:57:25', 0,
111, 58295, '3', '12/11/19 17:57:38', 13,
111, 58296, 'Q8', '12/11/19 17:57:38', 0,
111, 58297, '3', '12/11/19 17:57:50', 12,
111, 58298, 'Q9', '12/11/19 17:57:50', 0,
111, 58299, 'Timeout', '12/11/19 17:58:09', 19,
111, 58300, 'Q9', '12/11/19 17:58:09', 0,
111, 58301, '10', '12/11/19 17:58:32', 23,
111, 58302, 'Q10', '12/11/19 17:58:32', 0,
111, 58303, '1', '12/11/19 17:58:49', 17,
111, 58304, 'Q11', '12/11/19 17:58:49', 0,
111, 58307, '3', '12/11/19 17:59:02', 13,
111, 58308, 'Q12', '12/11/19 17:59:02', 0,
111, 58309, 'Timeout', '12/11/19 17:59:23', 21,
111, 58310, 'Q13', '12/11/19 17:59:23', 0,
111, 58311, 'Timeout', '12/11/19 17:59:44', 21,
111, 58312, 'Q13', '12/11/19 17:59:44', 0,
111, 58313, '4', '12/11/19 17:59:51', 7,
111, 58314, 'Q14', '12/11/19 17:59:51', 0,
111, 58318, '2', '12/11/19 18:00:19', 28
),
nrow=68, ncol=5, byrow=T,
dimnames=list(c(NULL), c("Respondent.ID", "order.ID", "Interaction", "Datetime", "difftime"))))
These types of problems can usually be solved with run length encoding.
Suppose in the code below that 'test' is the first dataset you posted and 'new' is the second dataset.
We would first need to figure out what entries in your data are start and stop positions or 'categories'.
start <- "Answer"
stop <- c("Timeout. Call failed.", "Completed")
cat <- ifelse(test$Interaction %in% start, "start",
ifelse(test$Interaction %in% stop, "stop", "other"))
Subsequently, we can run length encode cat, and get the start and stop positions, and based on these figure out what rows should be different.
rle <- rle(cat)
start_positions <- which(rle$values == "start")
stop_positions <- which(rle$values == "stop")
# Points that need to be changed are after starts and before stops
# The intersect ensures that we don't flag a 'start', 'other', 'start' sequence to be changed
change <- intersect(start_positions + 1, stop_positions - 1)
Finally, we need to encode the change as 1 and the rest as 0 and put it back into the data:
rle$values <- rep.int(0, length(rle$values))
rle$values[change] <- 1
test$Include <- inverse.rle(rle)
And we can check wether the result would be correct by making a confusion matrix:
> table(test = test$Include, new = new$Include)
new
test 0 1
0 16 0
1 0 71
EDIT: The following should work if there is no proper stop position. I've assumed that the data is ordered on Respondent.ID, so if that assumption is not true maybe this might not be the best way. It was not clear to me whether undocumented stops should be included or excluded. In the code below, they are included.
start <- "Answer"
stop <- c("Timeout. Call failed.", "Completed")
cat <- ifelse(test2$Interaction %in% start, "start",
ifelse(test2$Interaction %in% stop, "stop", "other"))
responder_rle <- rle(as.character(test2$Respondent.ID))
cat[cumsum(responder_rle$lengths)] <- "unplannedstop"
rle <- rle(cat)
start_positions <- which(rle$values == "start")
stop_positions <- which(rle$values == "stop")
unplanned_stop <- which(rle$values == "unplannedstop")
change <- c(intersect(start_positions + 1,
c(stop_positions, unplanned_stop) - 1),
unplanned_stop)
rle$values <- rep.int(0, length(rle$values))
rle$values[change] <- 1
test2$Include <- inverse.rle(rle)
If the unplanned stop should not be included, you could use the following for the change variable:
change <- intersect(start_positions + 1, c(stop_positions, unplanned_stop) - 1)
I'm developing a smartcard that communicates via middleware to a server. The server needs to authenticate itself to the smartcard. Part of this authentication process is a challenge that is encrypted with a sessionkey. The server needs to decrypt the challenge with the sessionkey. This happens with the following Cipher parameters:
Algorithm: AES
Mode: CBC
Padding: noPadding
The code for encryption on the card:
public class Card extends Applet {
private final static byte[] PUBLIC_KEY_EXPONENT_CA = new byte[] {(byte) 0x1, 0x00, 0x01};
private final static byte[] PUBLIC_KEY_MODULUS_CA = new byte[] { (byte) 0xC0, 0x16, 0x35, 0x2C, 0x2F, 0x0E, 0x6C, 0x6B, (byte) 0x96, (byte) 0x9F, 0x53, 0x0B, 0x00, (byte) 0xE9, 0x05, 0x71, (byte) 0xCB, 0x39, (byte) 0xD0, 0x23, (byte) 0xF0, (byte) 0x80, 0x45, 0x75, 0x00, (byte) 0xCD, (byte) 0x80, (byte) 0xC7, 0x4B, 0x3A, 0x1B, 0x61, (byte) 0x86, (byte) 0xF4, 0x44, (byte) 0xCD, 0x21, 0x63, (byte) 0xCB, 0x44, 0x63, (byte) 0x9D, (byte) 0x97, 0x4F, 0x40, (byte) 0xA4, 0x1E, 0x01, 0x16, 0x39, 0x1D, (byte) 0xE5, 0x67, (byte) 0xBE, 0x01, (byte) 0xC8, (byte) 0x82, 0x52, 0x0B, 0x13, 0x71, (byte) 0xF5, 0x38, 0x21 };
private final RSAPublicKey publicKeyCA;
private final static byte[] PUBLIC_KEY_EXPONENT_SP = new byte[3];
private final static byte[] PUBLIC_KEY_MODULUS_SP = new byte[64];
private final RSAPublicKey publicKeySP;
private final static byte[] KEY_SESSION = new byte[32];
private final static byte[] CHALLENGE_SESSION = new byte[32];
private static final short LENGTH_SIGNATURE = 64;
private static final short LENGTH_TIME = 8;
private final byte[] time = new byte[LENGTH_TIME];
private final byte[] lastValidationTime = new byte[LENGTH_TIME];
private final byte[] differenceTime = new byte[LENGTH_TIME];
private final byte[] signatureBytes = new byte[LENGTH_SIGNATURE];
private final byte[] hash = new byte[20];
private final byte[] storage = new byte[331];
private final RandomData random;
private final Signature signature;
private final MessageDigest digest;
private final Cipher cipherRSA;
private final Cipher cipherAES;
private Card() {
publicKeySP = (RSAPublicKey) KeyBuilder.buildKey(KeyBuilder.TYPE_RSA_PUBLIC, KeyBuilder.LENGTH_RSA_512, false);
signature = Signature.getInstance(Signature.ALG_RSA_SHA_PKCS1, false);
digest = MessageDigest.getInstance(MessageDigest.ALG_SHA, false);
cipherRSA = Cipher.getInstance(Cipher.ALG_RSA_PKCS1, false);
cipherAES = Cipher.getInstance(Cipher.ALG_AES_BLOCK_128_CBC_NOPAD, false);
random = RandomData.getInstance(RandomData.ALG_PSEUDO_RANDOM) ;
}
public void process(APDU apdu) throws ISOException {
byte[] buffer = apdu.getBuffer();
...
byte instruction = buffer[ISO7816.OFFSET_INS];
switch(instruction){
...
case INS_AUTHENTICATE_SP:
authenticateSP(apdu);
break;
default: ISOException.throwIt(ISO7816.SW_INS_NOT_SUPPORTED);
}
}
private void authenticateSP(APDU apdu) {
loadSPCertificate();
if (!publicKeySP.isInitialized()) {
ISOException.throwIt(ISO7816.SW_CONDITIONS_NOT_SATISFIED);
}
verifySPCertificate();
clear(KEY_SESSION);
clear(CHALLENGE_SESSION);
AESKey sessionKeySP = (AESKey) KeyBuilder.buildKey(KeyBuilder.TYPE_AES_TRANSIENT_DESELECT, KeyBuilder.LENGTH_AES_128, true);
random.generateData(KEY_SESSION, (short) 0, (short) KEY_SESSION.length);
sessionKeySP.setKey(KEY_SESSION, (short) 0);
byte[] sessionKeySPBytesEncrypted = new byte[64];
cipherRSA.init(publicKeySP, Cipher.MODE_ENCRYPT);
cipherRSA.doFinal(KEY_SESSION, (short) 0, (short) KEY_SESSION.length, sessionKeySPBytesEncrypted, (short) 0);
random.generateData(CHALLENGE_SESSION, (short) 0, (short) 32);
byte[] messageBytes = new byte[128];
byte[] messageBytesEncrypted = new byte[128];
Util.arrayCopy(CHALLENGE_SESSION, (short) 0, messageBytes, (short) 0, (short) 32);
Util.arrayCopy(storage, (short) 0, messageBytes, (short) 32, (short) 64);
Util.arrayFillNonAtomic(messageBytes, (short) 96, (short) 32, (byte) -1);
cipherAES.init(sessionKeySP, Cipher.MODE_ENCRYPT, new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, (short) 0, (short) 16);
cipherAES.doFinal(messageBytes, (short) 0, (short) messageBytes.length, messageBytesEncrypted, (short) 0);
apdu.setOutgoing();
apdu.setOutgoingLength((short) (sessionKeySPBytesEncrypted.length + messageBytesEncrypted.length));
apdu.sendBytesLong(sessionKeySPBytesEncrypted, (short) 0, (short) sessionKeySPBytesEncrypted.length);
apdu.sendBytesLong(messageBytesEncrypted, (short) 0, (short) messageBytesEncrypted.length);
}
private void verifySPCertificate() {
signature.init(publicKeyCA, Signature.MODE_VERIFY);
// The signature is used on the hash of the certificate without the signature bytes
Util.arrayCopy(storage, (short) 267, signatureBytes, (short) 0, (short) LENGTH_SIGNATURE);
digest.doFinal(storage, (short) 0, (short) 267, hash, (short) 0);
if (!signature.verify(hash, (short) 0, (short) hash.length, signatureBytes, (short) 0, (short) signatureBytes.length)) {
ISOException.throwIt(SW_VERIFY_CERTIFICATE_ERROR);
}
Util.arrayCopy(storage, (short) 259, time, (short) 0, (short) time.length);
byte result = BigIntNumber.compare(time, (byte) 259, lastValidationTime, (byte) 0, (byte) LENGTH_TIME);
if (result < 0) {
ISOException.throwIt(SW_VERIFY_CERTIFICATE_ERROR);
}
}
private void loadSPCertificate() {
Util.arrayCopy(storage, (short) 128, PUBLIC_KEY_EXPONENT_SP, (short) 0, (short) PUBLIC_KEY_EXPONENT_SP.length);
Util.arrayCopy(storage, (short) 131, PUBLIC_KEY_MODULUS_SP, (short) 0, (short) PUBLIC_KEY_MODULUS_SP.length);
publicKeySP.setExponent(PUBLIC_KEY_EXPONENT_SP, (short) 0, (short) PUBLIC_KEY_EXPONENT_SP.length);
publicKeySP.setModulus(PUBLIC_KEY_MODULUS_SP, (short) 0, (short) PUBLIC_KEY_MODULUS_SP.length);
}
}
sessionKeySP: [89, 93, -96, 94, 97, -115, -91, -90, 100, -17, 106, -109, 18, 114, -11, 3, -53, 116, 64, 100, -96, 38, 57, -21, 52, -111, 32, -97, 58, 39, 25, -128]
messageBytes: [32, -52, -49, -20, -85, 41, 125, 116, 45, -53, 53, 20, -53, -84, -128, -67, -114, -68, 66, -96, 67, 78, 61, 115, 112, 84, 97, 120, 79, 110, 87, 101, 98, 44, 32, 68, 67, 61, 101, 71, 111, 118, 101, 114, 110, 109, 101, 110, 116, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]
messageBytesEncrypted: [-15, -114, 51, 112, 32, -108, 72, 108, 4, -77, -126, 78, 89, -34, 5, 53, 51, -84, 99, 28, -16, -3, 50, 50, -31, 108, -98, -18, -113, -126, 18, 18, 72, 7, 49, -76, -7, 111, -33, 47, 18, -94, 82, -49, -75, 60, -113, -49, 80, 101, -71, -43, 29, 52, -33, 15, -71, 89, 7, 6, -15, 30, -69, 103, -14, 91, -116, -94, -99, -120, -110, 42, -76, 10, -17, -58, -77, 80, 57, 43, -128, -72, 104, -69, -9, -103, -123, -125, -98, 25, -116, 96, 13, -38, -78, 87, -18, 103, 0, 15, 0, -63, -19, 86, 31, -21, 41, -127, -127, 58, -104, 89, 41, 18, -70, -86, 98, -33, 58, 80, -12, -102, 7, -106, -14, -18, -69, 34]
The code for decryption on the server:
private byte[] verifyAuthenticationServiceProvider(byte[] encryptedData) throws Exception {
byte[] sessionKeyBytesEncrypted = new byte[64];
System.arraycopy(encryptedData, 0, sessionKeyBytesEncrypted, 0, 64);
byte[] messageBytesEncrypted = new byte[128];
System.arraycopy(encryptedData, 64, messageBytesEncrypted, 0, 128);
Cipher cipherRSA = Cipher.getInstance("RSA/ECB/PKCS1Padding");
cipherRSA.init(Cipher.DECRYPT_MODE, privateKey);
byte[] sessionKeyBytes = cipherRSA.doFinal(sessionKeyBytesEncrypted);
SecretKey sessionKey = new SecretKeySpec(sessionKeyBytes, 0, sessionKeyBytes.length, "AES");
Cipher cipherAES = Cipher.getInstance("AES/CBC/NoPadding");
IvParameterSpec ivspec = new IvParameterSpec(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 });
cipherAES.init(Cipher.DECRYPT_MODE, sessionKey, ivspec);
byte[] messageBytes = cipherAES.doFinal(messageBytesEncrypted);
byte[] challengeBytes = new byte[32];
byte[] subjectBytes = new byte[64];
System.arraycopy(messageBytes, 0, challengeBytes, 0, 32);
System.arraycopy(messageBytes, 32, subjectBytes, 0, 64);
...
}
sessionKey: [89, 93, -96, 94, 97, -115, -91, -90, 100, -17, 106, -109, 18, 114, -11, 3, -53, 116, 64, 100, -96, 38, 57, -21, 52, -111, 32, -97, 58, 39, 25, -128]
messageBytesEncrypted: [-15, -114, 51, 112, 32, -108, 72, 108, 4, -77, -126, 78, 89, -34, 5, 53, 51, -84, 99, 28, -16, -3, 50, 50, -31, 108, -98, -18, -113, -126, 18, 18, 72, 7, 49, -76, -7, 111, -33, 47, 18, -94, 82, -49, -75, 60, -113, -49, 80, 101, -71, -43, 29, 52, -33, 15, -71, 89, 7, 6, -15, 30, -69, 103, -14, 91, -116, -94, -99, -120, -110, 42, -76, 10, -17, -58, -77, 80, 57, 43, -128, -72, 104, -69, -9, -103, -123, -125, -98, 25, -116, 96, 13, -38, -78, 87, -18, 103, 0, 15, 0, -63, -19, 86, 31, -21, 41, -127, -127, 58, -104, 89, 41, 18, -70, -86, 98, -33, 58, 80, -12, -102, 7, -106, -14, -18, -69, 34]
messageBytes: [-74, -125, 17, -98, 106, -83, -100, 29, 89, 70, -87, -122, -104, 84, 36, -34, 87, -111, 10, 13, -12, -112, -48, -90, 127, 75, 95, 64, 11, -96, 25, 26, 92, -111, 10, 6, 93, 36, -95, 127, -50, 65, 15, -111, -82, 48, 2, 94, 8, -6, -102, -96, -43, -110, -57, -26, -79, -42, -57, 121, -110, -74, -108, 17, 44, -28, -43, -96, 79, 60, -91, 23, -67, -76, 10, -1, -57, -11, 10, 114, -74, 107, -99, -83, 38, 109, 99, -13, 44, 102, 112, 74, -13, -84, 60, -101, -19, -78, -50, 29, 50, 105, -10, -103, -107, -47, -60, 78, 61, -35, -12, 114, -21, -34, -110, -103, -91, -70, -73, 0, 118, 45, 113, 40, 99, -76, -8, 126]
Unfortunately, the decrypted messageBytes are not the same as the ones that were send on the card. Somehow, the decryption (on the server) is different from the encryption (on the card), but I can't understand why or how. My Cipher configuration is the same on the card, as on the server.
Note, that I checked that the used session key and IvParameters are correct both on card and server.
Also note, that I'm aware of the imposed security risks associated with this cryptographic code.
Update 1
Now, including the session key in bytes. Additionally I updated the size of the challenge to 32 bytes (because I need to encrypt it with AES afterwards again).
Update 2
Now, including all code on the card for the method authenticateSP.
Found the solution. The reason the decrypted bytes were not the same is that the session key on the card is set with a 256-bit (32-byte) array (KEYS_SESSION), while I specified that the AESkey should be build using a 128-bit key.
Lets say i have 4 matrices:
matrix<-list(
MA0275.1 = structure(c(0, 76, 0, 24, 0, 100, 0, 0, 0, 0,
100, 0, 0, 0, 100, 0, 72, 11, 16, 0, 53, 0, 0, 47), .Dim = c(4L,
6L), .Dimnames = list(c("A", "C", "G", "T"), NULL), id = "MA0275.1", accession = "ASG1"),
MA0276.1 = structure(c(0, 220, 8, 35, 0, 291, 0, 3, 61, 21,
133, 10, 58, 54, 101, 12, 130, 0, 54, 0, 0, 11, 8, 147, 33,
150, 8, 35, 80, 0, 92, 26, 0, 8, 249, 19, 0, 0, 256, 18), .Dim = c(4L,
10L), .Dimnames = list(c("A", "C", "G", "T"), NULL), id = "MA0276.1", accession = "ASH1"),
MA0277.1 = structure(c(63, 13, 13, 13, 100, 0, 0, 0, 100,
0, 0, 0, 88, 13, 0, 0, 75, 0, 25, 0, 0, 0, 100, 0, 78, 16,
3, 3, 81, 6, 6, 6, 63, 13, 13, 13), .Dim = c(4L, 9L), .Dimnames = list(
c("A", "C", "G", "T"), NULL), id = "MA0277.1", accession = "AZF1"),
MA0278.1 = structure(c(64, 217, 425, 292, 104, 552, 150,
192, 484, 111, 114, 288, 78, 401, 186, 333, 455, 51, 370,
122, 248, 34, 670, 46, 98, 724, 143, 33, 52, 918, 7, 20,
348, 346, 280, 24, 12, 3, 977, 6, 966, 5, 23, 4, 26, 6, 962,
4, 9, 10, 4, 975, 47, 930, 7, 15, 892, 42, 16, 49, 487, 123,
320, 68, 288, 140, 317, 254, 373, 110, 81, 434, 178, 367,
184, 268, 402, 140, 341, 114, 435, 229, 241, 94), .Dim = c(4L,
21L), .Dimnames = list(c("A", "C", "G", "T"), NULL), id = "MA0278.1", accession = "BAS1"))
These matrices are used for scoring sequences (affinity). The function i have enables me to give me 1 affinity score so only the first matrix is used.
But it would be nice if the score is defined per matrix in a data.frame(). I tried to do this with a for loop.
sequence<-"GCCTTTCCTTCTCTTCTCCGCGTGTGGAGGGAGCCAGCGCTTAGGCCGGAGCGAGCCTGGGGGCCGCCCGCCGTGAAGACATCGCGGGGACCGATTCACC"
for (i in matrix) {
score<-affinity(i,sequence)
}
This gives me a numeric value of 1 matrix. So the for loop doesn't work proper. I want it to give me all the affinity scores of every matrix.
The function for affinity:
affinity<-function (pwm, seq, Rmax = NULL, lambda = 0.7, pseudo.count = 1,
gc.content = 0.5, slide = FALSE)
{
if (is.null(seq) || is.na(seq) || mode(seq) != "character") {
stop("sequence must be a character string of length >= ncol(pwm)")
}
gap.pos = sapply(1:nchar(seq), function(i) substr(seq, i,
i) == "-")
seq = gsub("-", "", seq)
if (nchar(seq) < ncol(pwm)) {
stop("sequence must be a character string of length >= ncol(pwm)")
}
Rmax = ifelse(is.null(Rmax), exp(0.584 * ncol(pwm) - 5.66),
Rmax)
pwm = pwm + pseudo.count
at.content = 1 - gc.content
pwm = apply(pwm, 2, function(p) {
maxAT = max(p[c(1, 4)])
maxCG = max(p[c(2, 3)])
if (maxAT > maxCG) {
transformed = c(log(maxAT/p[1])/lambda, log((maxAT/at.content) *
(gc.content/p[2]))/lambda, log((maxAT/at.content) *
(gc.content/p[3]))/lambda, log(maxAT/p[4])/lambda)
}
else {
transformed = c(log((maxCG/gc.content) * (at.content/p[1]))/lambda,
log(maxCG/p[2])/lambda, log(maxCG/p[3])/lambda,
log((maxCG/gc.content) * (at.content/p[4]))/lambda)
}
if (maxAT == maxCG) {
transformed = log(maxAT/p)/lambda
}
return(transformed)
})
if (slide) {
z = .C("R_affinity", as.double(pwm), ncol(pwm), as.character(seq),
as.integer(nchar(seq)), as.double(Rmax), as.double(lambda),
double(length = nchar(seq) - ncol(pwm) + 1), PACKAGE = "tRap")
res = z[[7]]
gapped = numeric(length = nchar(seq) - ncol(pwm) + 1)
gapped[!gap.pos] = res
res = gapped
}
else {
z = .C("R_affinity_sum", as.double(pwm), ncol(pwm), as.character(seq),
as.integer(nchar(seq)), as.double(Rmax), as.double(lambda),
double(length = 1), PACKAGE = "tRap")
res = z[[7]]
}
return(res)
}
Try lapply
lapply(matrix, affinity, sequence)
PS : It's really bad idea to call your data matrix