Reproduce MySQL error: The server closed the connection (node.js)

Question

I\'m trying to reproduce a MySQL error I\'m seeing in my node.js app on EC2 with the node mysql library:

Connection lost: The server closed the connec

Answer 1

Using generic-pool, I wrote something that works locally. I guess I'll test it and see if it doesn't crash in bizarre manner on the server side.

// Test node connection pool stuff

// Create a MySQL connection pool with
// a max of 10 connections, a min of 2, and a 30 second max idle time
var poolModule = require('generic-pool');
var pool = poolModule.Pool({
    name     : 'mysql',
    create   : function(callback) {
        var Client = require('mysql').Client; // use node-mysql library in all it's dubious glory
        var c = new Client();
        c.user     = 'root';
        c.password = 'xxx';
        c.database = 'test';
        c.on('close', function (err) {
          console.log('mysqldb conn close');
        });
        c.on('error', function (err) {
          console.log('mysqldb error: ' + err);
        });

        // parameter order: err, resource
        // new in 1.0.6
        callback(null, c);
    },
    destroy  : function(client) { client.end(); },
    max      : 10,
    // optional. if you set this, make sure to drain() (see step 3)
    min      : 2, 
    // specifies how long a resource can stay idle in pool before being removed
    idleTimeoutMillis : 30000,
     // if true, logs via console.log - can also be a function
    log : true 
});

var http = require('http');
http.createServer(function (req, res) {

  // Get db conn
  pool.acquire(function(err, client) {
      if (err) {
          // handle error - this is generally the err from your
          // factory.create function 
          console.log('pool.acquire err: ' + err);
          res.writeHead(500, {'Content-Type': 'application/json'});
          out = {
            err: err
          }
          res.end(JSON.stringify(out));
      }
      else {
          client.query("select * from foo", [], function(err, results) {

              if(err){
                res.writeHead(500, {'Content-Type': 'application/json'});
                out = {
                  err: err
                }
                res.end(JSON.stringify(out));
              } else {
                res.writeHead(500, {'Content-Type': 'application/json'});
                out = {
                  results: results
                }
                res.end(JSON.stringify(out));
              }


              // return object back to pool
              pool.release(client);
          });
      }
  });
}).listen(9615);

Pretty please don't die at 4am for no apparent reason!

Answer 2

The solution is use pooling connection !

You can wrote code to handle connection manually, it works. However pooling is design for this, use pooling connection solved connection drop error.

var mysql = require('mysql');
var pool  = mysql.createPool({
    connectionLimit : 10,
     host            : 'example.org',
    user            : 'bob',
     password        : 'secret',
      database        : 'my_db'
    });

   pool.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
            if (error) throw error;
            console.log('The solution is: ', results[0].solution);
         });

pooling mysql connection

Answer 3

I was having similar problems and created a getConnection() wrapper function that checks the health of the mysql connection before returning it to the caller and re-establishes the connection as necessary. In my testing it has handled fatal and non-fatal connection issues transparently for the application. If the connection simply timed out, the application recovers without experiencing any errors. If there is a transient but fatal database connection problem, the application will resume functioning automatically as soon as database connectivity is available again.

As far as reproducing the problem for testing, add the following two lines to the my.ini or my.cnf file under the [mysqld] block:

interactive_timeout=30
wait_timeout=30

Here is the contents of a file I have named "database.js":

var mysql = require("mysql");
var CONFIG = require(__dirname + "/configuration");

module.exports.getConnection = function() {
    // Test connection health before returning it to caller.
    if ((module.exports.connection) && (module.exports.connection._socket)
            && (module.exports.connection._socket.readable)
            && (module.exports.connection._socket.writable)) {
        return module.exports.connection;
    }
    console.log(((module.exports.connection) ?
            "UNHEALTHY SQL CONNECTION; RE" : "") + "CONNECTING TO SQL.");
    var connection = mysql.createConnection({
        host     : CONFIG.db.host,
        user     : CONFIG.db.user,
        password : CONFIG.db.password,
        database : CONFIG.db.database,
        port     : CONFIG.db.port
    });
    connection.connect(function(err) {
        if (err) {
            console.log("SQL CONNECT ERROR: " + err);
        } else {
            console.log("SQL CONNECT SUCCESSFUL.");
        }
    });
    connection.on("close", function (err) {
        console.log("SQL CONNECTION CLOSED.");
    });
    connection.on("error", function (err) {
        console.log("SQL CONNECTION ERROR: " + err);
    });
    module.exports.connection = connection;
    return module.exports.connection;
}

// Open a connection automatically at app startup.
module.exports.getConnection();

// If you've saved this file as database.js, then get and use the
// connection as in the following example:
// var database = require(__dirname + "/database");
// var connection = database.getConnection();
// connection.query(query, function(err, results) { ....

Answer 4

Here's what I ended up using, and it worked pretty well. On the occasional connection lost/restart it recovered nicely. I have a database.js file which establishes connections and checks them periodically.

To make a request:

var conn = require('./database');
var sql = 'SELECT foo FROM bar;';
conn.query(sql, [userId, plugId], function (err, rows) {
   // logic
}

Here's my databbase.js

var mysql = require('mysql');
var Common = require('./common');
var conf = Common.conf;
var logger = Common.logger;

var connectionState = false;
var connection = mysql.createConnection({
  host: conf.db.hostname,
  user: conf.db.user,
  password: conf.db.pass,
  database: conf.db.schema,
  insecureAuth: true
});
connection.on('close', function (err) {
  logger.error('mysqldb conn close');
  connectionState = false;
});
connection.on('error', function (err) {
  logger.error('mysqldb error: ' + err);
  connectionState = false;
});

function attemptConnection(connection) {
  if(!connectionState){
    connection = mysql.createConnection(connection.config);
    connection.connect(function (err) {
      // connected! (unless `err` is set)
      if (err) {
        logger.error('mysql db unable to connect: ' + err);
        connectionState = false;
      } else {
        logger.info('mysql connect!');

        connectionState = true;
      }
    });
    connection.on('close', function (err) {
      logger.error('mysqldb conn close');
      connectionState = false;
    });
    connection.on('error', function (err) {
      logger.error('mysqldb error: ' + err);

      if (!err.fatal) {
        //throw err;
      }
      if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
        //throw err;
      } else {
        connectionState = false;
      }

    });
  }
}
attemptConnection(connection);

var dbConnChecker = setInterval(function(){
  if(!connectionState){
    logger.info('not connected, attempting reconnect');
    attemptConnection(connection);
  }
}, conf.db.checkInterval);

// Mysql query wrapper. Gives us timeout and db conn refreshal! 
var queryTimeout = conf.db.queryTimeout;
var query = function(sql,params,callback){
  if(connectionState) {
    // 1. Set timeout
    var timedOut = false;
    var timeout = setTimeout(function () {
      timedOut = true;
      callback('MySQL timeout', null);
    }, queryTimeout);

    // 2. Make query
    connection.query(sql, params, function (err, rows) {
      clearTimeout(timeout);
      if(!timedOut) callback(err,rows);
    });
  } else {
    // 3. Fail if no mysql conn (obviously)
    callback('MySQL not connected', null);
  }
}

// And we present the same interface as the node-mysql library!
// NOTE: The escape may be a trickier for other libraries to emulate because it looks synchronous
exports.query = query;
exports.escape = connection.escape;

Answer 5

Check out mysql pool feature in node-mysql

var mysql = require('mysql');
var pool  = mysql.createPool({
  host     : 'example.org',
  user     : 'bob',
  password : 'secret'
});

pool.getConnection(function(err, connection) {
  // connected! (unless `err` is set)
  connection.end();
});