MongoDB change stream timeouts if database is down for some time

问题

I am using mongoDB change stream in nodejs, everything works fine but if database is down has taken more than 10 5 seconds to get up change stream throws timeout error, here is my change stream watcher code

Service.prototype.watcher = function( db ){

let collection = db.collection('tokens');
let changeStream = collection.watch({ fullDocument: 'updateLookup' });
let resumeToken, newChangeStream;

changeStream.on('change', next => {
    resumeToken = next._id;
    console.log('data is ', JSON.stringify(next))
    changeStream.close();
    // console.log('resumeToken is ', JSON.stringify(resumeToken))
    newChangeStream = collection.watch({ resumeAfter : resumeToken });
    newChangeStream.on('change', next => {
        console.log('insert called ', JSON.stringify( next ))
    });
});

however on database end i have handled it, i.e if database is down or reconnected by using this code

 this.db.on('reconnected', function () {
    console.info('MongoDB reconnected!');
});
this.db.on('disconnected', function() {
    console.warn('MongoDB disconnected!');
});

but i am not able to handle change stream watcher to stop it when database is down and start it again when database is reconnected or if there is any other better way to do it ?

回答1:

What you want to do is to encapsulate the watch() call in a function. This function will then call itself on error, to rewatch the collection using a previously saved resume token. What is missing from the code you have is the error handler. For example:

const MongoClient = require('mongodb').MongoClient
const uri = 'mongodb://localhost:27017/test?replicaSet=replset'
var resume_token = null

run()

function watch_collection(con, db, coll) {
  console.log(new Date() + ' watching: ' + coll)
  con.db(db).collection(coll).watch({resumeAfter: resume_token})
    .on('change', data => {
      console.log(data)
      resume_token = data._id
    })
    .on('error', err => {
      console.log(new Date() + ' error: ' + err)
      watch_collection(con, coll)
    })
}

async function run() {
  con = await MongoClient.connect(uri, {"useNewUrlParser": true})
  watch_collection(con, 'test', 'test')
}

Note that watch_collection() contains the watch() method along with its handler. On change, it will print the change and store the resume token. On error, it will call itself to rewatch the collection again.

回答2:

This is the solution i developed, just add the stream.on(error) function so it will not crash when there is error, as restart the stream on database reconnect, also save resume token in file for every event, this is helpful when application is crashed or stopped and you run again and during that time if x number of records were added, so on application restart just get last resume token from file and start watcher from there it will get all records inserted after that and hence no record will be missed, here is code below

var rsToken ;
    try {
        rsToken = await this.getResumetoken()
    } catch (error) {
        rsToken = null ;
    }

    if (!rsToken)
        changeStream = collection.watch({ fullDocument: 'updateLookup' });
    else 
        changeStream = collection.watch({ fullDocument: 'updateLookup', resumeAfter : rsToken  });

    changeStream.on('change', next => {

        resumeToken = next._id;
        THIS.saveTokenInfile(resumeToken)

        cs_processor.process( next )


    });  
    changeStream.on('error', err => {
        console.log('changestream error ')
    })

来源：https://stackoverflow.com/questions/54614760/mongodb-change-stream-timeouts-if-database-is-down-for-some-time

标签

node.js

mongodb

changestream