Node.JS downloading hundreds of files simultaneously

问题

I am trying to download more that 100 files at the same time. But when I execute the downloading function my macbook freezes(unable to execute new tasks) in windows also no download(but doesn't freeze) and no download progress in both case(idle network).

Here is my download module:

var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');

router.get('/', function (req, res, next) {
    links.find({dlStatus: false}, function (err, docs) {

        if (err) {
            console.log(err);
            res.end();
        } else if (!docs) {
            console.log('No incomplete downloads!');
            res.end();
        } else {
            for (var i = 0; i < docs.length; i++) {

                //todo scraping


                var video = youtubedl(docs[i].url, [], {cwd: __dirname});

                // Will be called when the download starts.
                video.on('info', function (info) {
                    console.log('Download started');
                    console.log(info);
                });

                video.pipe(fs.createWriteStream('./downloads/' + docs[i].id + '-' + i + '.mp4'));
                video.on('complete', function complete(info) {
                    links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
                        if (err)console.log(err);
                        else console.log('Download completed!')
                    });
                });
            }
        }
    });
});

module.exports = router;

Now can anyone please help me here? I am using this module for downloading files.

回答1:

The solution is using async in this case.

Try it this way....with async.each()

var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');
var async = require('async')

router.get('/', function (req, res, next) {
    links.find({dlStatus: false}, function (err, docs) {

        if (err) {
            console.log(err);
            res.end();
        } else if (!docs) {
            console.log('No incomplete downloads!');
            res.end();
        } else {
            async.each(docs,function(doc,cb){
              var video = youtubedl(doc.url, [], {cwd: __dirname});

              // Will be called when the download starts.
              video.on('info', function (info) {
                  console.log('Download started');
                  console.log(info);
              });

              video.pipe(fs.createWriteStream('./downloads/' + docs.id + '-' + i + '.mp4'));
              video.on('complete', function complete(info) {
                  links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
                      if (err){
                        console.log(err);
                        cb(err);
                      }
                      else {
                        console.log('Download completed!');
                        cb()
                      }
                  });
              });

            },function(err){
              if(err)
                return console.log(err);
              console.log("Every thing is done,Here!!");
            })
        }
    });
});

module.exports = router;

And you can process every thing in batch too using async.eachLimits().

来源：https://stackoverflow.com/questions/38916666/node-js-downloading-hundreds-of-files-simultaneously

标签

node.js

express

youtube-dl