stream response from nodejs request to s3

问题

How do you use request to download contents of a file and directly stream it up to s3 using the aws-sdk for node?

The code below gives me Object #<Request> has no method 'read' which makes it seem like request does not return a readable stream...

var req = require('request');
var s3 = new AWS.S3({params: {Bucket: myBucket, Key: s3Key}});
var imageStream = req.get(url)
    .on('response', function (response) {
      if (200 == response.statusCode) {
        //imageStream should be read()able by now right?
        s3.upload({Body: imageStream, ACL: "public-read", CacheControl: 5184000}, function (err, data) {  //2 months
          console.log(err,data);
        });
      }
    });
});

Per the aws-sdk docs Body needs to be a ReadableStream object.

What am I doing wrong here?

This can be pulled off using the s3-upload-stream module, however I'd prefer to limit my dependencies.

回答1:

You want to use the response object if you're manually listening for the response stream:

var req = require('request');
var s3 = new AWS.S3({params: {Bucket: myBucket, Key: s3Key}});
var imageStream = req.get(url)
    .on('response', function (response) {
      if (200 == response.statusCode) {
        s3.upload({Body: response, ACL: "public-read", CacheControl: 5184000}, function (err, data) {  //2 months
          console.log(err,data);
        });
      }
    });
});

回答2:

Since I had the same problem as @JoshSantangelo (zero byte files on S3) with request@2.60.0 and aws-sdk@2.1.43, let me add an alternative solution using Node's own http module (caveat: simplified code from a real life project and not tested separately):

var http = require('http');

function copyToS3(url, key, callback) {
    http.get(url, function onResponse(res) {
        if (res.statusCode >= 300) {
            return callback(new Error('error ' + res.statusCode + ' retrieving ' + url));
        }
        s3.upload({Key: key, Body: res}, callback);
    })
    .on('error', function onError(err) {
        return callback(err);
    });
}

As far as I can tell, the problem is that request does not fully support the current Node streams API, while aws-sdk depends on it.

References:

• request issue about the readable event not working right
• generic issue for "new streams" support in request
• usage of the readable event in aws-sdk

来源：https://stackoverflow.com/questions/30902851/stream-response-from-nodejs-request-to-s3