Getting s3 object metadata then creating stream

问题

I'm downloading an object from s3 and creating a read stream object from it to process a video:

s3.getObject(params).createReadStream()

However, I need to get the metadata from it which is possible when i just get the object by accessing its 'metadata' property:

s3.getObject()

How would I either:

Get the object via s3.getObject(), grab the metadata from its metadata property, and then turn it into a read stream?

var stream = fs.createReadStream(response); isn't working - input must be a string

-- OR --

Get the stream via s3.getObject().createReadStream(), and extract the metadata from the stream?

To my knowledge metadata isn't passed within streams.

Tell me if my assumptions are wrong, but I am currently stuck with these two needs:

Getting the meta data
Making it a stream

回答1:

You can get the metadata via the request's httpHeaders event.

let fs = require('fs')
let aws = require('aws-sdk')
let s3 = new aws.S3()

let request = s3.getObject({
    Bucket: 'my-bucket',
    Key: 'my-key'
})

let stream

request.on('httpHeaders', (statusCode, httpHeaders) => {
    // object metadata is represented by any header in httpHeaders starting with 'x-amz-meta-'
    // you can use the stream object that this point
    stream.pipe(fs.createWriteStream('./somepath'))
    stream.on('end', () => { 
        console.log('were done')
    })
})

stream = request.createReadStream()

Alternatively you can also call s3.headObject to get the metadata without downloading the object and then download the object using s3.getObject

回答2:

So I kind of found a solution. This works for most files under 10 MB. If they are larger than that the buffer stream ends before the file is done being written. I've tried putting the bufferStream.end inside of the on finish function but then my call back doesnt go through....

function download(s3Event, srcKey, cb){

  console.log('Starting download');

  s3.getObject({
      Bucket: s3Event.bucket.name,
      Key: srcKey
  }, cb);
}



function writeToFile(data, cb){

  var dlFile = path.join(tempDir, 'download');

  console.log('data = ', data);
  console.log('data.Body = ', data.Body);
  var stream = bufferStream.pipe(fs.createWriteStream(dlFile)).on('finish', function () {
     console.log('finished writing stream');
     cb(null, data);
  });

  bufferStream.end(data.Body);
}


exports.handler = function(event, context) {

// Read options from the event.

console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));

var s3Event = event.Records[0].s3;
var srcKey = decodeURIComponent(s3Event.object.key);
var keyPrefix = srcKey.replace(/\.[^/.]+$/, '');         
var dstBucket = "jump-lambda";


async.waterfall([

    function (cb){
        download(s3Event, srcKey, cb);
    },

    function (data, cb){
        writeToFile(data, cb);
    },

    function (data, cb){
        fluentffmpegProcess(data, cb);
    },

    function (data, cb){
        transform(data, cb);
    },

    function (data, buffer, cb){
        thumbnailUpload(data, buffer, dstBucket, keyPrefix, cb);
    },

    function (data, cb){
        updateParse(data, srcKey, keyPrefix, cb);
    },

],

    function (err) {
        if (err) {
            console.error(
                'Unable to convert video to scene object, with error code: ' + err.description
            );
        } else {
            console.log(
                'Successfully created scene object, updated venue, and created thumbnail'
            );
        }
    }
);
};

来源：https://stackoverflow.com/questions/36920284/getting-s3-object-metadata-then-creating-stream

标签

node.js

amazon-s3

metadata

aws-lambda