How do you maintain the correct state when using Cloud Functions? They are not guaranteed to fire in the same order that they are called.
Here is a sequence of events:
A document is updated currentState: state1
A document is updated currentState: state2
The Cloud Function triggers the state2 update.
The Cloud Function triggers the state1 update.
If your application requires the carrying out of functions in the correct order of states, there's a problem.
Cloud Functions are not guaranteed to fire in order or only once. Therefore, you must make them idempotent.
You can resolve this in the following way:
Always use transactions to update the state, so that 2 clients don't try and change the state at the same time.
Create a state table which manages the state and runs functions based on the current state vs. the previous state.
Clients must not change the state to a value less than that which exists currently.
states.json
[
{"currentState": "state1", "action": "state2", "newStates": ["state2"]},
{"currentState": "state1", "action": "state3", "newStates": ["state2", "state3"]},
{"currentState": "state1", "action": "state4", "newStates": ["state2", "state3", "state4"]},
{"currentState": "state1", "action": "state5", "newStates": ["state2", "state3", "state4", "state5"]},
{"currentState": "state2", "action": "state3", "newStates": ["state3"]},
{"currentState": "state2", "action": "state4", "newStates": ["state3", "state4"]},
{"currentState": "state2", "action": "state5", "newStates": ["state3", "state4", "state5"]},
{"currentState": "state3", "action": "state4", "newStates": ["state4"]},
{"currentState": "state3", "action": "state5", "newStates": ["state4", "state5"]},
{"currentState": "state4", "action": "state5", "newStates": ["state5"]}
]
app.js
function processStates (beforeState, afterState) {
const states = require('../states');
let newStates;
// Check the states and set the new state
try {
newStates = states.filter(function(e) {return e.currentState == beforeState && e.action == afterState;})[0].newStates;
}
catch (err) {
newStates = null;
}
console.log(`newStates: ${newStates}`);
if (newStates) {
newStates.forEach(newState) {
// Process state change here
switch (newState) {
case 'state1': {
// Process state1 change
break;
}
case 'state2': {
// Process state2 change
break;
}
default: {
}
}
}
}
}
Once you have an array of states, you can iterate through the using something like forEach or map to process the required commands.
Related
I'm trying to add a custom resource via amplify. I proceeded to add the resource in CustomResources.json but it gives me an error:
CustomResourcesjson AWS::CloudFormation::Stack UPDATE_FAILED Thu Nov 24 2022 13:18:36…
🛑 An error occurred during the push operation: /
Resource is not in the state stackUpdateComplete
I also used the amplify add custom command in AWSCloudformation format but it gave me another type of error.
My goal is to reproduce this procedure written for serverless, but on amplify:
- type: AMAZON_DYNAMODB
name: likeMutation
config:
tableName: !Ref LikesTable
iamRoleStatements:
- Effect: Allow
Action: dynamodb:PutItem
Resource: !GetAtt LikesTable.Arn
- Effect: Allow
Action: dynamodb:UpdateItem
Resource:
- !GetAtt UsersTable.Arn
- !GetAtt TweetsTable.Arn
It is about creating a data source connected to a table but whose permissions allow you to perform operations on other tables.
The resource I added in the CustomResources.js file is:
"LikeMutationDataSource": {
"Type": "AWS::AppSync::DataSource",
"Properties": {
"ApiId": "f353mqkxyzcgncck6xqbtdlboe",
"DynamoDBConfig": {
"AwsRegion": "eu-west-1",
"TableName": "Likes-f353mqkxyzcgncck6xqbtdlboe-dev"
},
"Name": "LikeMutation",
"ServiceRoleArn": {
"Fn::GetAtt": [
"LikeMutationRoleMine"
]
},
"Type": "AMAZON_DYNAMODB"
}
}
},
"LikeMutationRoleMine": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "appsync.amazonaws.com"
}
}
],
"Version": "2012-10-17"
},
"RoleName": "LikeMutationRoleMine"
}
},
"LikeMutationPolicyMine": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": "dynamodb:PutItem",
"Resource": [
"arn:aws:dynamodb:eu-west-1:043166218277:table/Likes-f353mqkxyzcgncck6xqbtdlboe-dev",
"arn:aws:dynamodb:eu-west-1:043166218277:table/Likes-f353mqkxyzcgncck6xqbtdlboe-dev/*"
]
},
{
"Sid": "VisualEditor1",
"Effect": "Allow",
"Action": "dynamodb:UpdateItem",
"Resource": [
"arn:aws:dynamodb:eu-west-1:043166218277:table/User-f353mqkxyzcgncck6xqbtdlboe-dev",
"arn:aws:dynamodb:eu-west-1:043166218277:table/Post-f353mqkxyzcgncck6xqbtdlboe-dev",
"arn:aws:dynamodb:eu-west-1:043166218277:table/Post-f353mqkxyzcgncck6xqbtdlboe-dev/*",
"arn:aws:dynamodb:eu-west-1:043166218277:table/User-f353mqkxyzcgncck6xqbtdlboe-dev/*"
]
}
],
"Version": "2012-10-17"
},
"PolicyName": "LikeMutationPolicyMine",
"Roles": [
{
"Ref": "LikeMutationRoleMine"
}
]
}
Help me to implement this DataSource with amplify.
I've been looking for a solution for two days!
Thankyou all
The azure function is a .net core class library that will receive the message based on the namespace of the model being sent (in the filter as eventType) as an . All deployments are being done using arm templates, which is where this struggle is originating from. The function and eventgrid are deployed fine, but I don't know what i'm doing wrong with the subscription. If I create the subscription in the portal then the handler receives the message and displays traffic on the monitor. If I create the subscription as below then it appears exactly the same in the portal as the portal created one but nothing shows up in the monitor. Am I missing a resource or connection that still needs to be created? I read about system topics and how they're made implicitly in some instances but can be made explicitly, is that what I'm missing? This would be easier to debug if there was a place to export the template for those subscriptions but I don't see them.
Function handler
[FunctionName("FunctionName")]
public async Task Run([EventGridTrigger]EventGridEvent eventGridEvent)
{
...
}
}
eventgrid creation
{
"type": "Microsoft.EventGrid/topics",
"apiVersion": "2020-06-01",
"name": "[variables('EventGridName')]",
"location": "[resourceGroup().location]"
}
subscription creations
{
"name": "[concat(variables('eventSubscriptions')[copyIndex()].eventGridName, '/Microsoft.EventGrid/', variables('eventSubscriptions')[copyIndex()].name)]",
"type": "Microsoft.EventGrid/topics/providers/eventSubscriptions",
"apiVersion": "2020-01-01-preview",
"location": "[resourceGroup().location]",
"copy": {
"name": "subscriptionCopy",
"count": "[length(variables('eventSubscriptions'))]"
},
"properties": {
"topic": "[concat('/subscriptions/', subscription().subscriptionId,'/resourcegroups/', resourceGroup().name, '/providers/Microsoft.EventGrid/topics/', variables('eventSubscriptions')[copyIndex()].eventGridName)]",
"destination": {
"endpointType": "AzureFunction",
"properties": {
"resourceId": "[concat('/subscriptions/', subscription().subscriptionId,'/resourcegroups/', resourceGroup().name, '/providers/Microsoft.Web/sites/', variables('eventSubscriptions')[copyIndex()].functionApp, '/functions/' , variables('eventSubscriptions')[copyIndex()].functionName)]",
"maxEventsPerBatch": 1,
"preferredBatchSizeInKilobytes": 64
}
},
"filter": {
"includedEventTypes": [
"[variables('eventSubscriptions')[copyIndex()].eventType]"
]
},
"labels": [],
"eventDeliverySchema": "EventGridSchema"
},
"dependsOn": [
]
}
I am trying to deploy a glue crawler for an s3. Unfortunately I cant manage to find an appropriate IAM role that allows the crawler to run. The permissions I need are just to read/write to S3, and logs:PutLogsEvent, but somehow I am not getting it right.
Here is my code, it can be deployed but the crawler does not have permissions to run.
from aws_cdk import (
aws_events as events,
aws_lambda as lambda_,
aws_events_targets as targets,
aws_iam as iam,
aws_glue as glue,
core
)
class MyStack(core.Stack):
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# what should I put in the role exactly?
glue_role = iam.Role(
self, 'Role__arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole',
assumed_by=iam.ServicePrincipal('glue.amazonaws.com'),
)
glue_trigger = glue.CfnTrigger(self, "glue-daily-trigger",
name = "etl-trigger",
schedule = "cron(5 * * * ? *)", # every hour at X.05, every day
type="SCHEDULED",
actions=[
{
"jobName": "glue_crawler-daily"
}
],
start_on_creation=True
)
crawler_name = 'crawler_units_data'
glue_crawler = glue.CfnCrawler(
self, crawler_name,
name=crawler_name,
database_name='data_science',
role=glue_role.role_arn,#'arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole',
targets={"s3Targets": [{"path": "s3://random_s3/units/"}]},
)
glue_trigger.add_depends_on(glue_crawler)
I tried several things and translating code from javascript examples like this one but the methods being called from javascript do not map 100% with python.
This role (created from the GUI) works correctly and has 2 policies.
Policy to read and write from s3
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::random_s3/units*"
]
}
]
}
AWSGlueServicePolicy
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"glue:*",
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:ListAllMyBuckets",
"s3:GetBucketAcl",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeRouteTables",
"ec2:CreateNetworkInterface",
"ec2:DeleteNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"iam:ListRolePolicies",
"iam:GetRole",
"iam:GetRolePolicy",
"cloudwatch:PutMetricData"
],
"Resource": [
"*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:CreateBucket"
],
"Resource": [
"arn:aws:s3:::aws-glue-*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::aws-glue-*/*",
"arn:aws:s3:::*/*aws-glue-*/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject"
],
"Resource": [
"arn:aws:s3:::crawler-public*",
"arn:aws:s3:::aws-glue-*"
]
},
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": [
"arn:aws:logs:*:*:/aws-glue/*"
]
},
{
"Effect": "Allow",
"Action": [
"ec2:CreateTags",
"ec2:DeleteTags"
],
"Condition": {
"ForAllValues:StringEquals": {
"aws:TagKeys": [
"aws-glue-service-resource"
]
}
},
"Resource": [
"arn:aws:ec2:*:*:network-interface/*",
"arn:aws:ec2:*:*:security-group/*",
"arn:aws:ec2:*:*:instance/*"
]
}
]
}
As it turns out, I needed to pass the name and policy in a different way
glue_role = iam.Role(
self, 'glue_role_id2323',
role_name = 'Rolename',
assumed_by=iam.ServicePrincipal('glue.amazonaws.com'),
managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSGlueServiceRole')]
)
I have a SNS topic in account "A", which is a trigger of a Lambda function in that same account. This Lambda function sends a message to a private Slack channel.
This works fine, as long as the CloudWatch alarm is in the same account (Account A).
But I also want to do this from "Account B", but there I get:
{
"error": "Resource: arn:aws:cloudwatch:REGION:ACCOUNT_B:alarm:ALARM is not authorized to perform: SNS:Publish on resource: arn:aws:sns:REGION:ACCOUNT_A:TOPIC",
"actionState": "Failed",
"notificationResource": "arn:aws:sns:REGION:ACCOUNT_A:TOPIC",
"stateUpdateTimestamp": 1495732611020,
"publishedMessage": null
}
So how do I allow the CloudWatch Alarm ARN access to publish to the topic?
Trying to add the policy fails with:
Invalid parameter: Policy Error: PrincipalNotFound (Service: AmazonSNS; Status Code: 400; Error Code: InvalidParameter; Request ID: 7f5c202e-4784-5386-8dc5-718f5cc55725)
I see that someone else have/had the same problem (years ago!) at https://forums.aws.amazon.com/thread.jspa?threadID=143607, but it was never answered.
Update:
Trying to solve this, I'm now trying to use a local SNS topic, which then sends this to the remove account. However, I'm still getting:
"error": "Resource: arn:aws:cloudwatch:REGION:LOCAL_ACCOUNT:alarm:ALARM is not authorized to perform: SNS:Publish on resource: arn:aws:sns:REGION:LOCAL_ACCOUNT:TOPIC"
This, with this SNS policy:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowLambdaAccountToSubscribe",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::REMOTE_ACCOUNT:root"
},
"Action": [
"sns:Subscribe",
"sns:Receive"
],
"Resource": "arn:aws:sns:REGION:LOCAL_ACCOUNT:TOPIC"
},
{
"Sid": "AllowLocalAccountToPublish",
"Effect": "Allow",
"Principal": "*",
"Action": "sns:Publish",
"Resource": "arn:aws:sns:REGION:LOCAL_ACCOUNT:TOPIC",
"Condition": {
"StringEquals": {
"AWS:SourceAccount": "LOCAL_ACCOUNT"
}
}
}
]
}
If I manually send a message to the topic with the Publish to topic, I can see that it reaches the Lambda function, so everything except the CloudWatch access rights.
By means of trial-and-error, I discovered it was the Condition that didn't work. For some reason. Not sure why it didn't see the source account...
A more extensive policy made it work:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowLambdaAccountToSubscribe",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::REMOTE_ACCOUNT:root"
},
"Action": [
"sns:Subscribe",
"sns:Receive"
],
"Resource": "arn:aws:sns:REGION:LOCAL_ACCOUNT:TOPIC"
},
{
"Sid": "AllowLocalAccountToPublish",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "sns:Publish",
"Resource": "arn:aws:sns:REGION:LOCAL_ACCOUNT:TOPIC",
"Condition": {
"StringEquals": {
"AWS:SourceAccount": "LOCAL_ACCOUNT"
}
}
},
{
"Sid": "AllowCloudWatchAlarmsToPublish",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "sns:Publish",
"Resource": "arn:aws:sns:REGION:LOCAL_ACCOUNT:TOPIC",
"Condition": {
"ArnLike": {
"AWS:SourceArn": "arn:aws:cloudwatch:REGION:LOCAL_ACCOUNT:alarm:*"
}
}
}
]
}
I have get this error in wordpress: Error retrieving a list of your S3 buckets from AWS:Access Denied. I have write policy for IAM user. I don't know where I am doing wrong. Please help me.
My IAM policy is here:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Stmtxxxxxxxxxxx",
"Effect": "Allow",
"Action": [
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:ListBucketMultipartUploads"
],
"Resource": [
"arn:aws:s3:::bucketname/*"
]
}
]
}
First of all, in order to list all your buckets, you need to create a statement that will allow your IAM to perform "s3:ListAllMyBuckets" in the entire S3 account
{
"Effect": "Allow",
"Action": "s3:ListAllMyBuckets",
"Resource": "*",
"Condition": {}
}
Also, it's seems like you have trouble with bucket listing because the actions that you are trying to allow:
"Action": [
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:ListBucketMultipartUploads"
],
must be applied to the entire bucket:
"Resource": "arn:aws:s3:::bucketname",
while you are trying to allow this actions to the bucket's content:
"Resource": "arn:aws:s3:::bucketname/*",
Anyway, please try the policy below:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Stmtxxxxxxxxxxx",
"Effect": "Allow",
"Action": [
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::bucketname",
"Condition": {}
},
{
"Effect": "Allow",
"Action": "s3:ListAllMyBuckets",
"Resource": "*",
"Condition": {}
}
]
}
I tested it on my site and it works.
If you have any other questions, feel free to ask.
Thanks,
Vlad