Writing express middleware to get raw request body before body-parser

Question

I wrote an Express middleware to retrieve the raw body from the request, and I set it before body-parser middleware.

My custom middleware is calling req.setEncoding('utf8'), but this causes the following body-parser error:

Error: stream encoding should not be set

at readStream (/node_modules/body-parser/node_modules/raw-body/index.js:211:17) 
at getRawBody (/node_modules/body-parser/node_modules/raw-body/index.js:106:12)
at read (/node_modules/body-parser/lib/read.js:76:3)
at jsonParser (/node_modules/body-parser/lib/types/json.js:127:5)

Here is my code:

var express = require('express');
var bodyParser = require('body-parser')

function myMiddleware() {
  return function(req, res, next) {
    req.rawBody = '';
    req.setEncoding('utf8');

    req.on('data', function(chunk) {
      req.rawBody += chunk;
    });

    req.on('end', function() {
      next();
    });
  }
}

var app = express();
app.use(myMiddleware());
app.use(bodyParser.json());

var listener = app.listen(3000, function() {
});

app.get('/webhook/', function (req, res) {
  res.sendStatus(200);
});

Is there a way to unset the encoding? Is there another way to retireve the raw body, but still use body-parser after it?

Answer 1

It turns out that body-parser has a verify option to call a function when the request body has been read. The function receives the body as a buffer.

Here is an example:

var express = require('express');
var bodyParser = require('body-parser')

function verifyRequest(req, res, buf, encoding) {
  // The raw body is contained in 'buf'
  console.log( buf.toString( encoding ) );
};

var app = express();
var listener = app.listen(3000);

// Hook 'verifyRequest' with body-parser here.
app.use(bodyParser.json({ verify: verifyRequest }))

app.post('/webhook/', function (req, res) {
  res.status(200).send("done!");
});

Answer 2

You are calling next() inside "done", which means the stream has already been consumed. Instead, set up the handler for "data" then pass the request along using next(). The "done" event is likely being handled inside bodyParser so after it executes you have access to req.rawBody. If this was not the case you would add another middleware that calls next() inside a req.on('done') to hold the rest from processing until you have all the data.

// custom middleware - req, res, next must be arguments on the top level function
function myMiddleware(req, res, next) {
  req.rawBody = '';

  req.on('data', function(chunk) {
    req.rawBody += chunk;
  });

  // call next() outside of 'end' after setting 'data' handler
  next();  
}

// your middleware
app.use(myMiddleware);

// bodyparser
app.use(bodyParser.json())

// test that it worked
function afterMiddleware(req, res, next) {
  console.log(req.rawBody);
  next();  
}

app.use(afterMiddleware);

If you need to access the raw body you might also want to look into bodyParser.raw(). This will put the raw body into req.body, same as bodyParse.json() but can be made to run conditionally based on the content type - check out options.type.

Answer 3

I recommend a different approach, since your current approach actually consumes the message and makes it impossible for body-parser to read it (and there are a bunch of edge case bugs that spring up by calling next synchronously):

app.use(bodyParser.json());
app.use(bodyParser.text({type: '*/*'}));

This will read any application/json request as JSON, and everything else as text.

If you must have the JSON object in addition to the text, I recommend parsing it yourself:

app.use(bodyParser.text({type: '*/*'}));
app.use(myMiddleware);

function myMiddleware(req, res, next) {
    req.rawBody = req.body;
    if(req.headers['content-type'] === 'application/json') {
        req.body = JSON.parse(req.body);
    }
    next();
}