We are creating an S3 bucket using a CloudFormation template. I would like to associate (Add an event to S3 bucket) a Lambda function whenever a file is added to the S3 bucket.
How is it possible through CloudFormation templates. What are the properties which needs to be used in CloudFormation.
Here's a complete, self-contained CloudFormation template that demonstrates how to trigger a Lambda function whenever a file is added to an S3 bucket:
Description: Upload an object to an S3 bucket, triggering a Lambda event, returning the object key as a Stack Output.
Parameters:
Key:
Description: S3 Object key
Type: String
Default: test
Body:
Description: S3 Object body content
Type: String
Default: TEST CONTENT
BucketName:
Description: S3 Bucket name
Type: String
Resources:
Bucket:
Type: AWS::S3::Bucket
DependsOn: BucketPermission
Properties:
BucketName: !Ref BucketName
NotificationConfiguration:
LambdaConfigurations:
- Event: 's3:ObjectCreated:*'
Function: !GetAtt BucketWatcher.Arn
BucketPermission:
Type: AWS::Lambda::Permission
Properties:
Action: 'lambda:InvokeFunction'
FunctionName: !Ref BucketWatcher
Principal: s3.amazonaws.com
SourceAccount: !Ref "AWS::AccountId"
SourceArn: !Sub "arn:aws:s3:::${BucketName}"
BucketWatcher:
Type: AWS::Lambda::Function
Properties:
Description: Sends a Wait Condition signal to Handle when invoked
Handler: index.handler
Role: !GetAtt LambdaExecutionRole.Arn
Code:
ZipFile: !Sub |
exports.handler = function(event, context) {
console.log("Request received:\n", JSON.stringify(event));
var responseBody = JSON.stringify({
"Status" : "SUCCESS",
"UniqueId" : "Key",
"Data" : event.Records[0].s3.object.key,
"Reason" : ""
});
var https = require("https");
var url = require("url");
var parsedUrl = url.parse('${Handle}');
var options = {
hostname: parsedUrl.hostname,
port: 443,
path: parsedUrl.path,
method: "PUT",
headers: {
"content-type": "",
"content-length": responseBody.length
}
};
var request = https.request(options, function(response) {
console.log("Status code: " + response.statusCode);
console.log("Status message: " + response.statusMessage);
context.done();
});
request.on("error", function(error) {
console.log("send(..) failed executing https.request(..): " + error);
context.done();
});
request.write(responseBody);
request.end();
};
Timeout: 30
Runtime: nodejs4.3
Handle:
Type: AWS::CloudFormation::WaitConditionHandle
Wait:
Type: AWS::CloudFormation::WaitCondition
Properties:
Handle: !Ref Handle
Timeout: 300
S3Object:
Type: Custom::S3Object
Properties:
ServiceToken: !GetAtt S3ObjectFunction.Arn
Bucket: !Ref Bucket
Key: !Ref Key
Body: !Ref Body
S3ObjectFunction:
Type: AWS::Lambda::Function
Properties:
Description: S3 Object Custom Resource
Handler: index.handler
Role: !GetAtt LambdaExecutionRole.Arn
Code:
ZipFile: !Sub |
var response = require('cfn-response');
var AWS = require('aws-sdk');
var s3 = new AWS.S3();
exports.handler = function(event, context) {
console.log("Request received:\n", JSON.stringify(event));
var responseData = {};
if (event.RequestType == 'Create') {
var params = {
Bucket: event.ResourceProperties.Bucket,
Key: event.ResourceProperties.Key,
Body: event.ResourceProperties.Body
};
s3.putObject(params).promise().then(function(data) {
response.send(event, context, response.SUCCESS, responseData);
}).catch(function(err) {
console.log(JSON.stringify(err));
response.send(event, context, response.FAILED, responseData);
});
} else if (event.RequestType == 'Delete') {
var deleteParams = {
Bucket: event.ResourceProperties.Bucket,
Key: event.ResourceProperties.Key
};
s3.deleteObject(deleteParams).promise().then(function(data) {
response.send(event, context, response.SUCCESS, responseData);
}).catch(function(err) {
console.log(JSON.stringify(err));
response.send(event, context, response.FAILED, responseData);
});
} else {
response.send(event, context, response.SUCCESS, responseData);
}
};
Timeout: 30
Runtime: nodejs4.3
LambdaExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal: {Service: [lambda.amazonaws.com]}
Action: ['sts:AssumeRole']
Path: /
ManagedPolicyArns:
- "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
Policies:
- PolicyName: S3Policy
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- 's3:PutObject'
- 'S3:DeleteObject'
Resource: !Sub "arn:aws:s3:::${BucketName}/${Key}"
Outputs:
Result:
Value: !GetAtt Wait.Data
S3Object in that template? I do not see that it is refered anywhere?
S3Object and S3ObjectFunction automatically creates an object in the S3 bucket to trigger the Lambda function, just for testing. Normally this would be triggered by objects you upload manually, so it's not needed in your own implementation.
You need a NotificationConfiguration property in your CloudFormation template. Unfortunately, it seems to require the bucket to already exist. To get around this, you can create an initial stack, then update it with the NotificationConfiguration. For example:
// template1.json
{
"AWSTemplateFormatVersion": "2010-09-09",
"Parameters": {
"mylambda": {
"Type": "String"
}
},
"Resources": {
"bucketperm": {
"Type": "AWS::Lambda::Permission",
"Properties" : {
"Action": "lambda:InvokeFunction",
"FunctionName": {"Ref": "mylambda"},
"Principal": "s3.amazonaws.com",
"SourceAccount": {"Ref": "AWS::AccountId"},
"SourceArn": { "Fn::Join": [":", [
"arn", "aws", "s3", "" , "", {"Ref" : "mybucket"}]]
}
}
},
"mybucket": {
"Type": "AWS::S3::Bucket"
}
}
}
// template2.json -- adds the NotificationConfiguration
{
"AWSTemplateFormatVersion": "2010-09-09",
"Parameters": {
"mylambda": {
"Type": "String"
}
},
"Resources": {
"bucketperm": {
"Type": "AWS::Lambda::Permission",
"Properties" : {
"Action": "lambda:InvokeFunction",
"FunctionName": {"Ref": "mylambda"},
"Principal": "s3.amazonaws.com",
"SourceAccount": {"Ref": "AWS::AccountId"},
"SourceArn": { "Fn::Join": [":", [
"arn", "aws", "s3", "" , "", {"Ref" : "mybucket"}]]
}
}
},
"mybucket": {
"Type": "AWS::S3::Bucket",
"Properties": {
"NotificationConfiguration": {
"LambdaConfigurations": [
{
"Event" : "s3:ObjectCreated:*",
"Function" : {"Ref": "mylambda"}
}
]
}
}
}
}
}
You can use the AWS CLI tool to create the stack like this:
$ aws cloudformation create-stack --stack-name mystack --template-body file://template1.json --parameters ParameterKey=mylambda,ParameterValue=<lambda arn>
# wait until stack is created
$ aws cloudformation update-stack --stack-name mystack --template-body file://template2.json --parameters ParameterKey=mylambda,ParameterValue=<lambda arn>
i have added below bucket perm along with notificationconfiguration in cloudformation which is used to create S3 bucket..it worked !!
"bucketperm": {
"Type": "AWS::Lambda::Permission",
"Properties": {
"Action": "lambda:invokeFunction",
"FunctionName": "<arnvalue>",
"Principal": "s3.amazonaws.com"
}
}
Yes, it's possible through Cloudformation, and what you need to configure are:
1) AWS::S3::Bucket resource and,
2) NotificationConfiguration configuration (use LambdaConfigurations in this case) for the s3 resource above.
Related documentation that you need:
It is clearly stated in AWS docs that AWS::S3::Bucket is used to create a resource, If we have a bucket that exists already we can not modify it to add NotificationConfiguration. So S3 bucket must not exist for above template to work. Let CloudFormation creates all resources including S3 bucket.
