Using node-csv and meteor-file to import CSV into a collection

Question

Been struggling for several hours now trying to import CSV, uploaded from client using meteor-file and converted to CSV using node-csv serv

Answer 1

I wanted to find a solution that didn't load the entire CSV file into memory, for use with large datasets. Here's my solution that uses Meteor.bindEnvironment, along with node-csv, to parse a CSV file into a Meteor Collection.

Thanks to the folks on #meteor for the help.

var csv = Meteor.require('CSV'); 
var fs = Meteor.require('fs');
var path = Npm.require('path');

function loadData() {
  var basepath = path.resolve('.').split('.meteor')[0];

  csv().from.stream(
    fs.createReadStream(basepath+'server/data/enron_data.csv'),
      {'escape': '\\'})
    .on('record', Meteor.bindEnvironment(function(row, index) {
      Emails.insert({
        'sender_id': row[0]
        // etc.
        })
      }, function(error) {
          console.log('Error in bindEnvironment:', error);
      }
    ))
    .on('error', function(err) {
      console.log('Error reading CSV:', err);
    })
    .on('end', function(count) {
      console.log(count, 'records read');
    });

}

Answer 2

So it turns out because CSV() uses a callback and runs Asynchronous code, I needed to use a 'Future'. For more explaination see http://gist.io/3443021

here's my working code:

Meteor.methods({
     'uploadFile': function (file) {

      Future = Npm.require('fibers/future');

      console.log(file.name+'\'+file.type+'\'+file.size);                       

      file.save('/home/russell/tmp',{});
      var buffer = new Buffer(file.data);


      // Set up the Future
      var fut = new Future(); 

      // Convert buffer (a CSV file) to an array
      CSV().from(
                   buffer.toString(),
                   {comment: '#', delimiter: ',', quote: ''} 
                )
           .to.array( function(data){

                                       var newRecords=[];

                                       for(var row=0; row<data.length; row++) {
                                         console.log(data[row]);
                                         newRecord = {
                                                       'firstname': data[row][0],
                                                       'lastname': data[row][1],
                                                       'email': data[row][2],
                                                       'emailshort': data[row][3],
                                                       'emailmain': data[row][4],
                                                       'domain': data[row][5]
                                                     };

                                         //console.log(newRecord);
                                         newRecords.push(newRecord);
                                  }

                                  // at the end of the CSV callback
                                  // return newRecords via the Future
                                  fut['return'](newRecords);
    } );


    // Wait for the results of the conversion
    results = fut.wait();
    console.log('results================');
    console.log(results);

    // now insert the new records from the file into our collectiion
    if (results.length) {
        for(i in results) {
            reas.insert(results[i]);
        }
    }

    console.log('reas now looks like =====================');
    console.log(reas.find({}).fetch());

} // uploadFile

});