Append data to an S3 object

Question

Let\'s say that I have a machine that I want to be able to write to a certain log file stored on an S3 bucket.

So, the machine needs to have writing abilities to tha

Answer 1

Objects on S3 are not append-able. You have 2 solutions in this case:

1. copy all S3 data to a new object, append the new content and write back to S3.

function writeToS3(input) {
    var content;
    var getParams = {
        Bucket: 'myBucket', 
        Key: "myKey"
    };

    s3.getObject(getParams, function(err, data) {
        if (err) console.log(err, err.stack);
        else {
            content = new Buffer(data.Body).toString("utf8");
            content = content + '\n' + new Date() + '\t' + input;
            var putParams = {
                Body: content,
                Bucket: 'myBucket', 
                Key: "myKey",
                ACL: "public-read"
             };

            s3.putObject(putParams, function(err, data) {
                if (err) console.log(err, err.stack); // an error occurred
                else     {
                    console.log(data);           // successful response
                }
             });
        }
    });  
}

2. Second option is to use Kinesis Firehose. This is fairly straightforward. You need to create your firehose delivery stream and link the destination to S3 bucket. That's it!

function writeToS3(input) {
    var content = "\n" + new Date() + "\t" + input;
    var params = {
      DeliveryStreamName: 'myDeliveryStream', /* required */
      Record: { /* required */
        Data: new Buffer(content) || 'STRING_VALUE' /* Strings will be Base-64 encoded on your behalf */ /* required */
      }
    };

    firehose.putRecord(params, function(err, data) {
      if (err) console.log(err, err.stack); // an error occurred
      else     console.log(data);           // successful response
    }); 
}