How to send a zipped file to S3 bucket from Apex? - http

Question

Folks,
I am trying to move data to s3 from Salesforce using apex class. I have been told by the data manager to send the data in zip/gzip format to the S3 bucket for storage cost savings.
I have simply tried to do a request.setCompressed(true); as I've read it compresses the body before sending it to the endpoint. Code below:
HttpRequest request = new HttpRequest();
request.setEndpoint('callout:'+DATA_NAMED_CRED+'/'+URL+'/'+generateUniqueTimeStampforSuffix());
request.setMethod('PUT');
request.setBody(JSON.serialize(data));
request.setCompressed(true);
request.setHeader('Content-Type','application/json');
But no matter what I always receive this:
<Error><Code>XAmzContentSHA256Mismatch</Code><Message>The provided 'x-amz-content-sha256' header does not match what was computed.</Message><ClientComputedContentSHA256>fd31b2b9115ef77e8076b896cb336d21d8f66947210ffcc9c4d1971b2be3bbbc</ClientComputedContentSHA256><S3ComputedContentSHA256>1e7f2115e60132afed9e61132aa41c3224c6e305ad9f820e6893364d7257ab8d</S3ComputedContentSHA256>
I have tried multiple headers too, like setting the content type to gzip/zip, etc.
Any pointers in the right direction would be appreciated.

Answer 1

I had a good amount of headaches attempting to do a similar thing. I feel your pain.
The following code has worked for us using lambda functions; you can try modifying it and see what happens.
public class AwsApiGateway {
// Things we need to know about the service. Set these values in init()
String host, payloadSha256;
String resource;
String service = 'execute-api';
String region;
public Url endpoint;
String accessKey;
String stage;
string secretKey;
HttpMethod method = HttpMethod.XGET;
// Remember to set "payload" here if you need to specify a body
// payload = Blob.valueOf('some-text-i-want-to-send');
// This method helps prevent leaking secret key,
// as it is never serialized
// Url endpoint;
// HttpMethod method;
Blob payload;
// Not used externally, so we hide these values
Blob signingKey;
DateTime requestTime;
Map<String, String> queryParams = new map<string,string>(), headerParams = new map<string,string>();
void init(){
if (payload == null) payload = Blob.valueOf('');
requestTime = DateTime.now();
createSigningKey(secretKey);
}
public AwsApiGateway(String resource){
this.stage = AWS_LAMBDA_STAGE
this.resource = '/' + stage + '/' + resource;
this.region = AWS_REGION;
this.endpoint = new Url(AWS_ENDPOINT);
this.accessKey = AWS_ACCESS_KEY;
this.secretKey = AWS_SECRET_KEY;
}
// Make sure we can't misspell methods
public enum HttpMethod { XGET, XPUT, XHEAD, XOPTIONS, XDELETE, XPOST }
public void setMethod (HttpMethod method){
this.method = method;
}
public void setPayload (string payload){
this.payload = Blob.valueOf(payload);
}
// Add a header
public void setHeader(String key, String value) {
headerParams.put(key.toLowerCase(), value);
}
// Add a query param
public void setQueryParam(String key, String value) {
queryParams.put(key.toLowerCase(), uriEncode(value));
}
// Create a canonical query string (used during signing)
String createCanonicalQueryString() {
String[] results = new String[0], keys = new List<String>(queryParams.keySet());
keys.sort();
for(String key: keys) {
results.add(key+'='+queryParams.get(key));
}
return String.join(results, '&');
}
// Create the canonical headers (used for signing)
String createCanonicalHeaders(String[] keys) {
keys.addAll(headerParams.keySet());
keys.sort();
String[] results = new String[0];
for(String key: keys) {
results.add(key+':'+headerParams.get(key));
}
return String.join(results, '\n')+'\n';
}
// Create the entire canonical request
String createCanonicalRequest(String[] headerKeys) {
return String.join(
new String[] {
method.name().removeStart('X'), // METHOD
new Url(endPoint, resource).getPath(), // RESOURCE
createCanonicalQueryString(), // CANONICAL QUERY STRING
createCanonicalHeaders(headerKeys), // CANONICAL HEADERS
String.join(headerKeys, ';'), // SIGNED HEADERS
payloadSha256 // SHA256 PAYLOAD
},
'\n'
);
}
// We have to replace ~ and " " correctly, or we'll break AWS on those two characters
string uriEncode(String value) {
return value==null? null: EncodingUtil.urlEncode(value, 'utf-8').replaceAll('%7E','~').replaceAll('\\+','%20');
}
// Create the entire string to sign
String createStringToSign(String[] signedHeaders) {
String result = createCanonicalRequest(signedHeaders);
return String.join(
new String[] {
'AWS4-HMAC-SHA256',
headerParams.get('date'),
String.join(new String[] { requestTime.formatGMT('yyyyMMdd'), region, service, 'aws4_request' },'/'),
EncodingUtil.convertToHex(Crypto.generateDigest('sha256', Blob.valueof(result)))
},
'\n'
);
}
// Create our signing key
void createSigningKey(String secretKey) {
signingKey = Crypto.generateMac('hmacSHA256', Blob.valueOf('aws4_request'),
Crypto.generateMac('hmacSHA256', Blob.valueOf(service),
Crypto.generateMac('hmacSHA256', Blob.valueOf(region),
Crypto.generateMac('hmacSHA256', Blob.valueOf(requestTime.formatGMT('yyyyMMdd')), Blob.valueOf('AWS4'+secretKey))
)
)
);
}
// Create all of the bits and pieces using all utility functions above
public HttpRequest createRequest() {
init();
payloadSha256 = EncodingUtil.convertToHex(Crypto.generateDigest('sha-256', payload));
setHeader('date', requestTime.formatGMT('yyyyMMdd\'T\'HHmmss\'Z\''));
if(host == null) {
host = endpoint.getHost();
}
setHeader('host', host);
HttpRequest request = new HttpRequest();
request.setMethod(method.name().removeStart('X'));
if(payload.size() > 0) {
setHeader('Content-Length', String.valueOf(payload.size()));
request.setBodyAsBlob(payload);
}
String finalEndpoint = new Url(endpoint, resource).toExternalForm(),
queryString = createCanonicalQueryString();
if(queryString != '') {
finalEndpoint += '?'+queryString;
}
request.setEndpoint(finalEndpoint);
for(String key: headerParams.keySet()) {
request.setHeader(key, headerParams.get(key));
}
String[] headerKeys = new String[0];
String stringToSign = createStringToSign(headerKeys);
request.setHeader(
'Authorization',
String.format(
'AWS4-HMAC-SHA256 Credential={0}, SignedHeaders={1},Signature={2}',
new String[] {
String.join(new String[] { accessKey, requestTime.formatGMT('yyyyMMdd'), region, service, 'aws4_request' },'/'),
String.join(headerKeys,';'), EncodingUtil.convertToHex(Crypto.generateMac('hmacSHA256', Blob.valueOf(stringToSign), signingKey))}
));
system.debug(json.serializePretty(request.getEndpoint()));
return request;
}
// Actually perform the request, and throw exception if response code is not valid
public HttpResponse sendRequest(Set<Integer> validCodes) {
HttpResponse response = new Http().send(createRequest());
if(!validCodes.contains(response.getStatusCode())) {
system.debug(json.deserializeUntyped(response.getBody()));
}
return response;
}
// Same as above, but assume that only 200 is valid
// This method exists because most of the time, 200 is what we expect
public HttpResponse sendRequest() {
return sendRequest(new Set<Integer> { 200 });
}
// TEST METHODS
public static string getEndpoint(string attribute){
AwsApiGateway api = new AwsApiGateway(attribute);
return api.createRequest().getEndpoint();
}
public static string getEndpoint(string attribute, map<string, string> params){
AwsApiGateway api = new AwsApiGateway(attribute);
for (string key: params.keySet()){
api.setQueryParam(key, params.get(key));
}
return api.createRequest().getEndpoint();
}
public class EndpointConfig {
string resource;
string attribute;
list<object> items;
map<string,string> params;
public EndpointConfig(string resource, string attribute, list<object> items){
this.items = items;
this.resource = resource;
this.attribute = attribute;
}
public EndpointConfig setQueryParams(map<string,string> parameters){
params = parameters;
return this;
}
public string endpoint(){
if (params == null){
return getEndpoint(resource);
} else return getEndpoint(resource + '/' + attribute, params);
}
public SingleRequestMock mockResponse(){
return new SingleRequestMock(200, 'OK', json.serialize(items), null);
}
}
}