Update:
I believe that this could be caused by the fact that I'm using the body parser provided by express. Could this be messing with the stream that multiparty is trying to parse?
I'm basing my solution on this answer.
What I'm trying to do: Stream a file from a client browser straight to S3 with my NodeJS server acting as a proxy (for security purposes). I don't want the file to touch the server's filesystem to avoid that bottleneck.
I'm getting the following error:
events.js:72
throw er; // Unhandled 'error' event
^
Error: stream ended unexpectedly
at Form.<anonymous> (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/index.js:619:24)
at Form.EventEmitter.emit (events.js:117:20)
at finishMaybe (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:443:14)
at endWritable (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:452:3)
at Form.Writable.end (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:419:5)
at onend (_stream_readable.js:457:10)
at process._tickCallback (node.js:415:13)
I've looked at the code and can't quite seem to understand what's causing the issue. I'm using angular-file-upload because I'm using angular on the front end. Here's what the request looks like:
Request URL:http://local.bucketstreams.com:3000/upload/image
Request Headers CAUTION: Provisional headers are shown.
Accept:application/json, text/plain, */*
Cache-Control:no-cache
Content-Type:multipart/form-data; boundary=----WebKitFormBoundaryNKuH2H9IUB7kvmea
Origin:http://local.bucketstreams.com:3000
Pragma:no-cache
Referer:http://local.bucketstreams.com:3000/getting-started
User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36
Request Payload
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="type"
profile
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="user"
pie
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="file0"; filename="Screen Shot 2014-02-18 at 10.54.06 PM.png"
Content-Type: image/png
------WebKitFormBoundaryNKuH2H9IUB7kvmea--
And here's what my code looks like:
var ErrorController = require('../controller/ErrorController');
var AuthenticationController = require('../controller/AuthenticationController');
var logger = require('winston');
var http = require('http');
var util = require('util');
var multiparty = require('multiparty');
var knox = require('knox');
var Batch = require('batch');
var s3Client = knox.createClient({
secure: false,
key: process.env.S3_KEY,
secret: process.env.S3_SECRET,
bucket: process.env.S3_BUCKET_IMAGES
});
var Writable = require('readable-stream').Writable;
util.inherits(ByteCounter, Writable);
function ByteCounter(options) {
Writable.call(this, options);
this.bytes = 0;
}
ByteCounter.prototype._write = function(chunk, encoding, cb) {
this.bytes += chunk.length;
cb();
};
var supportedTypes = {
profile: true,
post: true
};
module.exports = function(app) {
app.post('/upload/image', AuthenticationController.checkAuthenticated, function(req, res) {
var type = req.body.type;
var userId = req.user._id;
if (!supportedTypes[type]) {
return ErrorController.sendErrorJson(res, 401, 'Unsupported image upload type: ' + type);
}
var headers = {
'x-amz-acl': 'public-read'
};
var form = new multiparty.Form();
var batch = new Batch();
batch.push(function(cb) {
form.on('field', function(name, value) {
if (name === 'path') {
var destPath = value;
if (destPath[0] !== '/') destPath = '/' + destPath;
cb(null, destPath);
}
});
});
batch.push(function(cb) {
form.on('part', function(part) {
if (! part.filename) return;
cb(null, part);
});
});
batch.end(function(err, results) {
if (err) throw err;
form.removeListener('close', onEnd);
var destPath = '/' + userId + results[0];
var part = results[1];
var counter = new ByteCounter();
part.pipe(counter); // need this until knox upgrades to streams2
headers['Content-Length'] = part.byteCount;
s3Client.putStream(part, destPath, headers, function(err, s3Response) {
if (err) throw err;
res.statusCode = s3Response.statusCode;
s3Response.pipe(res);
console.log('https://s3.amazonaws.com/' + process.env.S3_BUCKET_IMAGES + destPath);
});
part.on('end', function() {
console.log('part end');
console.log('size', counter.bytes);
});
});
form.on('close', function(error) {
logger.error(error);
return ErrorController.sendErrorJson(res, 500, 'There was a problem uploading the file.');
});
form.parse(req);
});
};
It looks like the part that is blowing up is multiparty and I've looked into that code a little bit to no avail. I'm not certain if I'm making the request incorrectly or if there's something wrong with my server code. I don't think it has anything to do with my S3 bucket, but I suppose it could be that as well.
Anyway, any tips are welcome.
There is an example for S3 on the node-multiparty source.
https://github.com/andrewrk/node-multiparty/tree/master/examples
var http = require('http'),
util = require('util'),
multiparty = require('../'),
knox = require('knox'),
Batch = require('batch'),
PORT = process.env.PORT || 27372
var s3Client = knox.createClient({
secure: false,
key: process.env.S3_KEY,
secret: process.env.S3_SECRET,
bucket: process.env.S3_BUCKET,
});
var server = http.createServer(function(req, res) {
if (req.url === '/') {
res.writeHead(200, {
'content-type': 'text/html'
});
res.end(
'<form action="/upload" enctype="multipart/form-data" method="post">' +
'<input type="text" name="path"><br>' +
'<input type="file" name="upload"><br>' +
'<input type="submit" value="Upload">' +
'</form>'
);
} else if (req.url === '/upload') {
var headers = {
'x-amz-acl': 'public-read',
};
var form = new multiparty.Form();
var batch = new Batch();
batch.push(function(cb) {
form.on('field', function(name, value) {
if (name === 'path') {
var destPath = value;
if (destPath[0] !== '/') destPath = '/' + destPath;
cb(null, destPath);
}
});
});
batch.push(function(cb) {
form.on('part', function(part) {
if (!part.filename) return;
cb(null, part);
});
});
batch.end(function(err, results) {
if (err) throw err;
form.removeListener('close', onEnd);
var destPath = results[0],
part = results[1];
headers['Content-Length'] = part.byteCount;
s3Client.putStream(part, destPath, headers, function(err, s3Response) {
if (err) throw err;
res.statusCode = s3Response.statusCode;
s3Response.pipe(res);
console.log("https://s3.amazonaws.com/" + process.env.S3_BUCKET + destPath);
});
});
form.on('close', onEnd);
form.parse(req);
} else {
res.writeHead(404, {
'content-type': 'text/plain'
});
res.end('404');
}
function onEnd() {
throw new Error("no uploaded file");
}
});
server.listen(PORT, function() {
console.info('listening on http://0.0.0.0:' + PORT + '/');
});
````
The crux to the solution is to not use the (apparently deprecated) bodyParser(). I'm not certain what it does, but it screws up the form's parts that multiparty uses. So instead, if you have the same problem I had, instead of using bodyParser(), use the things you need explicitely (for example):
app.use(express.urlencoded());
app.use(express.json());
And then for your multipart stuff, just use multiparty to parse the body yourself. The author of multiparty gives more info on the subject.
来源：https://stackoverflow.com/questions/21873561/streaming-file-to-s3-error-stream-ended-unexpectedly