How do I read the contents of a Node.js stream into a string variable?

Question

I\'m hacking on a Node program that uses smtp-protocol to capture SMTP emails and act on the mail data. The library provides the mail data as a stream, and I don\'t know how

Answer 1

Easy way with the popular (over 5m weekly downloads) and lightweight get-stream library:

https://www.npmjs.com/package/get-stream

const fs = require('fs');
const getStream = require('get-stream');

(async () => {
    const stream = fs.createReadStream('unicorn.txt');
    console.log(await getStream(stream)); //output is string
})();

Answer 2

Another way would be to convert the stream to a promise (refer to the example below) and use then (or await) to assign the resolved value to a variable.

function streamToString (stream) {
  const chunks = []
  return new Promise((resolve, reject) => {
    stream.on('data', chunk => chunks.push(chunk))
    stream.on('error', reject)
    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
  })
}

const result = await streamToString(stream)

Answer 3

I'm using usually this simple function to transform a stream into a string:

function streamToString(stream, cb) {
  const chunks = [];
  stream.on('data', (chunk) => {
    chunks.push(chunk.toString());
  });
  stream.on('end', () => {
    cb(chunks.join(''));
  });
}

Usage example:

let stream = fs.createReadStream('./myFile.foo');
streamToString(stream, (data) => {
  console.log(data);  // data is now my string variable
});

Answer 4

Using the quite popular stream-buffers package which you probably already have in your project dependencies, this is pretty straightforward:

// imports
const { WritableStreamBuffer } = require('stream-buffers');
const { promisify } = require('util');
const { createReadStream } = require('fs');
const pipeline = promisify(require('stream').pipeline);

// sample stream
let stream = createReadStream('/etc/hosts');

// pipeline the stream into a buffer, and print the contents when done
let buf = new WritableStreamBuffer();
pipeline(stream, buf).then(() => console.log(buf.getContents().toString()));

Answer 5

What about something like a stream reducer ?

Here is an example using ES6 classes how to use one.

var stream = require('stream')

class StreamReducer extends stream.Writable {
  constructor(chunkReducer, initialvalue, cb) {
    super();
    this.reducer = chunkReducer;
    this.accumulator = initialvalue;
    this.cb = cb;
  }
  _write(chunk, enc, next) {
    this.accumulator = this.reducer(this.accumulator, chunk);
    next();
  }
  end() {
    this.cb(null, this.accumulator)
  }
}

// just a test stream
class EmitterStream extends stream.Readable {
  constructor(chunks) {
    super();
    this.chunks = chunks;
  }
  _read() {
    this.chunks.forEach(function (chunk) { 
        this.push(chunk);
    }.bind(this));
    this.push(null);
  }
}

// just transform the strings into buffer as we would get from fs stream or http request stream
(new EmitterStream(
  ["hello ", "world !"]
  .map(function(str) {
     return Buffer.from(str, 'utf8');
  })
)).pipe(new StreamReducer(
  function (acc, v) {
    acc.push(v);
    return acc;
  },
  [],
  function(err, chunks) {
    console.log(Buffer.concat(chunks).toString('utf8'));
  })
);

Answer 6

In my case, the content type response headers was Content-Type: text/plain. So, I've read the data from Buffer like:

let data = [];
stream.on('data', (chunk) => {
 console.log(Buffer.from(chunk).toString())
 data.push(Buffer.from(chunk).toString())
});