DynamoDB ADD Transaction Fails Intermittently for no Apparent Reason - amazon-dynamodb

Question

I am having an intermittent problem with DynamoDB with a writing a bunch of entries. I cannot see what the problem and the weird thing the problem occurs in a place where there is no ConditionExpression. This exception happens about 1 in 10 times.
docClient.transactWrite(transaction).promise() with:
{
"TransactItems": [
{
"Update": {
"TableName": "protect-dev-stepfunction-ScanAggregation",
"Key": {
"pk": "#ScanCount#",
"sort": "#Total"
},
"UpdateExpression": "ADD queued :queued, runningFiles :runningFiles",
"ExpressionAttributeValues": {
":queued": 1,
":runningFiles": 0
},
"ReturnValues": "UPDATED_NEW"
}
},
{
"Update": {
"TableName": "protect-dev-stepfunction-ScanAggregation",
"Key": {
"pk": "#ScanCount#",
"sort": "#Tenant#pwc"
},
"UpdateExpression": "SET isTenant = :isTenant ADD queued :queued, runningFiles :runningFiles",
"ExpressionAttributeValues": {
":queued": 1,
":runningFiles": 0,
":isTenant": true
},
"ReturnValues": "UPDATED_NEW"
}
},
{
"Put": {
"TableName": "protect-dev-stepfunction-Jobs",
"Item": {
"createdAt": "2020-05-11 05:25:58.770369",
"scanId": "1161",
"tenantId": "pwc",
"TaskToken": "AAAAKgAAAAIAAAAAAAAAAb67i0NKEMIMzPxx5gzBSG8BhJwwVpTcsItdwbbkeOzPqefqQdqUu7yL3MMKLX3BR6ATsol4d60iFoOhDbR6WKmtukG5VH0rUHbxf8NMGcnZ+xhvD1bTyJbh1z2KEAHjQixI8sP/fhH065sygK9s8PS9XyvU2L/mEWcrn0ZGBLk+HDhHE2cwscYK2GYeKdVVthzCwgZ220tyYBHZBt3BXfKmjs3/8Xru8E0lk345Ub/mbalcQMTGPIYskk24UtnJNJJ5Qw9KSDBWl4e0DD0XULfcwWqsLZhVPwXS/e4j2XGE9k6XexusItElQAyopZ4njcU9YEPcchfsynDn0WptXpEMmO1gDYqE+oDaASazGIC3Oms7PhpD0XB9oApO84NvSik1q9VtXrnO5FEcBi5wG4GhYk2216Ga9lhc+S0luUIHfQQrkHtFPsw4uCaXTjTpCjCNG3wZqBjbS3GDbDnsW0yey74pc1FeEdB46os6v1OtGhjQBBbIlBS7L8t7HqNOtDrnBsqDort2u7Td0LUyp9YEksacSs4DlxKHtQ9aysxJVpFJVTGnrAuxfXcafo63R6aU8O764+6Nbm/QBWUJOdvkSQ5IaSDevqeaeLzHgeqY4KiXfCaRt2WYXljOTNucp5N8yr6ZAYGDGRrIIRiF5DY=",
"priority": "3",
"fileId": "3179",
"pk": "pwc",
"sort": "3-1161_3179",
"scanStatus": "SUBMITTED",
"rjk": "3-1161_3179"
},
"ConditionExpression": "attribute_not_exists(pk) and attribute_not_exists(sort)"
}
}
]
}{
"TransactItems": [
{
"Update": {
"TableName": "protect-dev-stepfunction-ScanAggregation",
"Key": {
"pk": "#ScanCount#",
"sort": "#Total"
},
"UpdateExpression": "ADD queued :queued, runningFiles :runningFiles",
"ExpressionAttributeValues": {
":queued": 1,
":runningFiles": 0
},
"ReturnValues": "UPDATED_NEW"
}
},
{
"Update": {
"TableName": "protect-dev-stepfunction-ScanAggregation",
"Key": {
"pk": "#ScanCount#",
"sort": "#Tenant#pwc"
},
"UpdateExpression": "SET isTenant = :isTenant ADD queued :queued, runningFiles :runningFiles",
"ExpressionAttributeValues": {
":queued": 1,
":runningFiles": 0,
":isTenant": true
},
"ReturnValues": "UPDATED_NEW"
}
},
{
"Put": {
"TableName": "protect-dev-stepfunction-Jobs",
"Item": {
"createdAt": "2020-05-11 05:25:58.770369",
"scanId": "1161",
"tenantId": "pwc",
"TaskToken": "AAAAKgAAAAIAAAAAAAAAAb67i0NKEMIMzPxx5gzBSG8BhJwwVpTcsItdwbbkeOzPqefqQdqUu7yL3MMKLX3BR6ATsol4d60iFoOhDbR6WKmtukG5VH0rUHbxf8NMGcnZ+xhvD1bTyJbh1z2KEAHjQixI8sP/fhH065sygK9s8PS9XyvU2L/mEWcrn0ZGBLk+HDhHE2cwscYK2GYeKdVVthzCwgZ220tyYBHZBt3BXfKmjs3/8Xru8E0lk345Ub/mbalcQMTGPIYskk24UtnJNJJ5Qw9KSDBWl4e0DD0XULfcwWqsLZhVPwXS/e4j2XGE9k6XexusItElQAyopZ4njcU9YEPcchfsynDn0WptXpEMmO1gDYqE+oDaASazGIC3Oms7PhpD0XB9oApO84NvSik1q9VtXrnO5FEcBi5wG4GhYk2216Ga9lhc+S0luUIHfQQrkHtFPsw4uCaXTjTpCjCNG3wZqBjbS3GDbDnsW0yey74pc1FeEdB46os6v1OtGhjQBBbIlBS7L8t7HqNOtDrnBsqDort2u7Td0LUyp9YEksacSs4DlxKHtQ9aysxJVpFJVTGnrAuxfXcafo63R6aU8O764+6Nbm/QBWUJOdvkSQ5IaSDevqeaeLzHgeqY4KiXfCaRt2WYXljOTNucp5N8yr6ZAYGDGRrIIRiF5DY=",
"priority": "3",
"fileId": "3179",
"pk": "pwc",
"sort": "3-1161_3179",
"scanStatus": "SUBMITTED",
"rjk": "3-1161_3179"
},
"ConditionExpression": "attribute_not_exists(pk) and attribute_not_exists(sort)"
}
}
]
}
And I get this stacktrace:
TransactionCanceledException: Transaction cancelled, please refer cancellation reasons for specific reasons [TransactionConflict, TransactionConflict, None]
at Request.extractError (/var/runtime/node_modules/aws-sdk/lib/protocol/json.js:51:27)
at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
at Request.emit (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:78:10)
at Request.emit (/var/runtime/node_modules/aws-sdk/lib/request.js:683:14)
at Request.transition (/var/runtime/node_modules/aws-sdk/lib/request.js:22:10)
at AcceptorStateMachine.runTo (/var/runtime/node_modules/aws-sdk/lib/state_machine.js:14:12)
at /var/runtime/node_modules/aws-sdk/lib/state_machine.js:26:10
at Request.<anonymous> (/var/runtime/node_modules/aws-sdk/lib/request.js:38:9)
at Request.<anonymous> (/var/runtime/node_modules/aws-sdk/lib/request.js:685:12)
at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:116:18)
message:
'Transaction cancelled, please refer cancellation reasons for specific reasons [TransactionConflict, TransactionConflict, None]',
code: 'TransactionCanceledException',
time: 2020-05-11T05:26:00.972Z,
requestId: 'ABKB5RCR30SDASOL3KGS5KEBUJVV4KQNSO5AEMVJF66Q9ASUAAJG',
statusCode: 400,
retryable: false,
retryDelay: 32.29530018146246 }

Answer 1

The error says it's a "TransactionConflict", which the documentation says can happen in these scenarios:
A PutItem, UpdateItem, or DeleteItem request for an item conflicts with an ongoing TransactWriteItems request that includes the same item.
An item within a TransactWriteItems request is part of another ongoing TransactWriteItems request.
An item within a TransactGetItems request is part of an ongoing TransactWriteItems, BatchWriteItem, PutItem, UpdateItem, or DeleteItem request.
In your case, it looks like the 2nd scenario. How much concurrency is there on your records?