DynamoDB data to Redshift - amazon-dynamodb

Question

I'm using Dynamo DB streams + Lamdba as trigger to invoke kinesis which puts my Dynamo DB data to Redshift.
Could someone suggest a way to load Dynamo DB data to Redshift in different regions using dynamo streams.

Answer 1

I had written one program that can move the data from Dynamo db to Redshift but that is not working without streams, You can take a look of code and see whether this helps your case or you get any Idea corresponding to this.
1.Creating the Connection with Redshift.
2.Create the Prepeared Statement for insertion into Redshift.
3.Fetching the data from Dynamo in batches using pagination.
4.Inserting the batch data into Resdhift in batches.
public void createConnectionWithRedshift() {
final String DB_URL = "jdbc:redshift://ao.cepuhmobd.us-west-2.redshift.amazonaws.com:5439/events";
// final String DB_URL = args[0];
// Database credentials
final String USER = "abc";
final String PASS = "abc";
Connection conn = null;
try {
// STEP 3: Open a connection
System.out.println("Connecting to database...");
conn = DriverManager.getConnection(DB_URL, USER, PASS);
// createNewTable(conn);
// STEP 4: Execute a query
preparedStatement = conn.prepareStatement("insert into Events " + "(Vin,timestamp,eventtype,source,data)" + "VALUES (?,?,?,?,?)");
} catch (SQLException se) {
se.printStackTrace();
}
}// end main
public void replicateDynamoToRedshidt(int pages, int batchSize, int scanSize)
throws TableNeverTransitionedToStateException, InterruptedException {
createConnectionWithRedshift();//Redshift Connection
for (int i = 0; i < pages; i = i + 1) {
List<EventLogEntity> results = findAll(new PageRequest(i, batchSize));//Fetching the data from Dynamo in batches
List<HeadUnitData> headUnitDataList = headUnitEvents(results);
for (int j = 0; j < headUnitDataList.size(); j++) {
HeadUnitData headUnitData = headUnitDataList.get(j);
insertData(headUnitData.getVin(), headUnitData.getType(), headUnitData.getSource(), headUnitData.getData());//Inserting the data into Redshidt in batches
}
try {
preparedStatement.executeBatch();
System.out.println("Inserted in Database : " + results.size());
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}