ReadStream: Internal buffer does not fill up anymore

问题

I have a fs.ReadStream object that points to a pretty big file. Now I would like to read 8000 bytes from the ReadStream, but the internal buffer is only 6000 bytes. So my approach would be to read those 6000 bytes and wait for the internal buffer to fill up again by using a while-loop that checks whether the internal buffer length is not 0 anymore.

Something like this:

BinaryObject.prototype.read = function(length) {
  var value;

  // Check whether we have enough data in the internal buffer
  if (this.stream._readableState.length < length) {
    // Not enough data - read the full internal buffer to
    // force the ReadStream to fill it again.
    value = this.read(this.stream._readableState.length);
    while (this.stream._readableState.length === 0) {
      // Wait...?
    }
    // We should have some more data in the internal buffer
    // here... Read the rest and add it to our `value` buffer
    // ... something like this:
    //
    // value.push(this.stream.read(length - value.length))
    // return value
  } else {
    value = this.stream.read(length);
    this.stream.position += length;
    return value;
  }
};

The problem is, that the buffer is not filled anymore - the script will just idle in the while loop.

What is the best approach to do this?

回答1:

It's quite simple. You don't need to do any buffering on your side:

var fs = require('fs'),
    rs = fs.createReadStream('/path/to/file');

var CHUNK_SIZE = 8192;

rs.on('readable', function () {
  var chunk;
  while (null !== (chunk = rs.read(CHUNK_SIZE))) {
    console.log('got %d bytes of data', chunk.length);
  }
});

rs.on('end', function () {
  console.log('end');
});

If CHUNK_SIZE is larger than the internal buffer, node will return null and buffer some more before emitting readable again. You can even configure the initial size of the buffer by passing:

var  rs = fs.createReadStream('/path/to/file', {highWatermark: CHUNK_SIZE});

回答2:

Below is the sample for reading file in streams.

var fs = require('fs'),
readStream = fs.createReadStream(srcPath);

readStream.on('data', function (chunk) {
  console.log('got %d bytes of data', chunk.length);
});

readStream.on('readable', function () {
  var chunk;
  while (null !== (chunk = readStream.read())) {
   console.log('got %d bytes of data', chunk.length);
  }
});

readStream.on('end', function () {
  console.log('got all bytes of data');
});

来源：https://stackoverflow.com/questions/19425817/readstream-internal-buffer-does-not-fill-up-anymore

标签

node.js

stream

buffer