Looping through files in a folder Node.JS

Question

I am trying to loop through and pick up files in a directory, but I have some trouble implementing it. How to pull in multiple files and then move them to another folder?

Answer 1

fs.readdir(path[, options], callback) (which Mikey A. Leonetti used in his answer) and its variants (fsPromises.readdir(path[, options]) and fs.readdirSync(path[, options])) each reads all of a directory's entries into memory at once. That's good for most cases, but if the directory has very many entries and/or you want to lower your application's memory footprint, you could instead iterate over the directory's entries one at a time.

Asynchronously

Directories are async iterable, so you could do something like this:

const fs = require('fs')

async function ls(path) {
  const dir = await fs.promises.opendir(path)
  for await (const dirent of dir) {
    console.log(dirent.name)
  }
}

ls('.').catch(console.error)

Or, you could use dir.read() and/or dir.read(callback) directly.

Synchronously

Directories aren't sync iterable, but you could use dir.readSync() directly. For example:

const fs = require('fs')

const dir = fs.opendirSync('.')
let dirent
while ((dirent = dir.readSync()) !== null) {
  console.log(dirent.name)
}
dir.closeSync()

Or, you could make directories sync iterable. For example:

const fs = require('fs')

function makeDirectoriesSyncIterable() {
  const p = fs.Dir.prototype
  if (p.hasOwnProperty(Symbol.iterator)) { return }
  const entriesSync = function* () {
    try {
      let dirent
      while ((dirent = this.readSync()) !== null) { yield dirent }
    } finally { this.closeSync() }
  }
  if (!p.hasOwnProperty(entriesSync)) { p.entriesSync = entriesSync }
  Object.defineProperty(p, Symbol.iterator, {
    configurable: true,
    enumerable: false,
    value: entriesSync,
    writable: true
  })
}
makeDirectoriesSyncIterable()

And then, you could do something like this:

const dir = fs.opendirSync('.')
for (const dirent of dir) {
  console.log(dirent.name)
}

---

Note: "In busy processes, use the asynchronous versions of these calls. The synchronous versions will block the entire process until they complete, halting all connections."

References:

• Node.js Documentation: File System: Class fs.Dir
• Node.js source code: fs.Dir
• GitHub: nodejs/node: Issues: streaming / iterative fs.readdir #583

Answer 2

The answers provided are for a single folder. Here is an asynchronous implementation for multiple folders where all the folders are processed simultaneously but the smaller folders or files gets completed first.

Please comment if you have any feedback

Asynchronously Multiple Folders

const fs = require('fs')
const util = require('util')
const path = require('path')

// Multiple folders list
const in_dir_list = [
  'Folder 1 Large',
  'Folder 2 Small', // small folder and files will complete first
  'Folder 3 Extra Large'
]

// BEST PRACTICES: (1) Faster folder list For loop has to be outside async_capture_callback functions for async to make sense
//                 (2) Slower Read Write or I/O processes best be contained in an async_capture_callback functions because these processes are slower than for loop events and faster completed items get callback-ed out first 

for (i = 0; i < in_dir_list.length; i++) {
  var in_dir = in_dir_list[i]

  // function is created (see below) so each folder is processed asynchronously for readFile_async that follows
  readdir_async_capture(in_dir, function(files_path) {
    console.log("Processing folders asynchronously ...")

    for (j = 0; j < files_path.length; j++) {
      file_path = files_path[j]
      file = file_path.substr(file_path.lastIndexOf("/") + 1, file_path.length)

      // function is created (see below) so all files are read simultaneously but the smallest file will be completed first and get callback-ed first 
      readFile_async_capture(file_path, file, function(file_string) {
        try {
          console.log(file_path)
          console.log(file_string)
        } catch (error) {
          console.log(error)
          console.log("System exiting first to catch error if not async will continue...")
          process.exit()
        }
      })
    }
  })
}

// fs.readdir async_capture function to deal with asynchronous code above
function readdir_async_capture(in_dir, callback) {
  fs.readdir(in_dir, function(error, files) {
    if (error) { return console.log(error) }
    files_path = files.map(function(x) { return path.join(in_dir, x) })
    callback(files_path)
  })
}

// fs.readFile async_capture function to deal with asynchronous code above
function readFile_async_capture(file_path, file, callback) {
  fs.readFile(file_path, function(error, data) {
    if (error) { return console.log(error) }
    file_string = data.toString()
    callback(file_string)
  })
}

Answer 3

Older answer with callbacks

You want to use the fs.readdir function to get the directory contents and the fs.rename function to actually do the renaming. Both these functions have synchronous versions if you need to wait for them to finishing before running the code afterwards.

I wrote a quick script that does what you described.

var fs = require('fs');
var path = require('path');
// In newer Node.js versions where process is already global this isn't necessary.
var process = require("process");

var moveFrom = "/home/mike/dev/node/sonar/moveme";
var moveTo = "/home/mike/dev/node/sonar/tome"

// Loop through all the files in the temp directory
fs.readdir(moveFrom, function (err, files) {
  if (err) {
    console.error("Could not list the directory.", err);
    process.exit(1);
  }

  files.forEach(function (file, index) {
    // Make one pass and make the file complete
    var fromPath = path.join(moveFrom, file);
    var toPath = path.join(moveTo, file);

    fs.stat(fromPath, function (error, stat) {
      if (error) {
        console.error("Error stating file.", error);
        return;
      }

      if (stat.isFile())
        console.log("'%s' is a file.", fromPath);
      else if (stat.isDirectory())
        console.log("'%s' is a directory.", fromPath);

      fs.rename(fromPath, toPath, function (error) {
        if (error) {
          console.error("File moving error.", error);
        } else {
          console.log("Moved file '%s' to '%s'.", fromPath, toPath);
        }
      });
    });
  });
});

Tested on my local machine.

node testme.js 
'/home/mike/dev/node/sonar/moveme/hello' is a file.
'/home/mike/dev/node/sonar/moveme/test' is a directory.
'/home/mike/dev/node/sonar/moveme/test2' is a directory.
'/home/mike/dev/node/sonar/moveme/test23' is a directory.
'/home/mike/dev/node/sonar/moveme/test234' is a directory.
Moved file '/home/mike/dev/node/sonar/moveme/hello' to '/home/mike/dev/node/sonar/tome/hello'.
Moved file '/home/mike/dev/node/sonar/moveme/test' to '/home/mike/dev/node/sonar/tome/test'.
Moved file '/home/mike/dev/node/sonar/moveme/test2' to '/home/mike/dev/node/sonar/tome/test2'.
Moved file '/home/mike/dev/node/sonar/moveme/test23' to '/home/mike/dev/node/sonar/tome/test23'.
Moved file '/home/mike/dev/node/sonar/moveme/test234' to '/home/mike/dev/node/sonar/tome/test234'.

Update: fs.promises functions with async/await

Inspired by ma11hew28's answer (shown here), here is the same thing as above but with the async functions in fs.promises. As noted by ma11hew28, this may have memory limitations versus fs.promises.opendir added in v12.12.0.

Quick code below.

//jshint esversion:8
//jshint node:true
const fs = require( 'fs' );
const path = require( 'path' );

const moveFrom = "/tmp/movefrom";
const moveTo = "/tmp/moveto";

// Make an async function that gets executed immediately
(async ()=>{
    // Our starting point
    try {
        // Get the files as an array
        const files = await fs.promises.readdir( moveFrom );

        // Loop them all with the new for...of
        for( const file of files ) {
            // Get the full paths
            const fromPath = path.join( moveFrom, file );
            const toPath = path.join( moveTo, file );

            // Stat the file to see if we have a file or dir
            const stat = await fs.promises.stat( fromPath );

            if( stat.isFile() )
                console.log( "'%s' is a file.", fromPath );
            else if( stat.isDirectory() )
                console.log( "'%s' is a directory.", fromPath );

            // Now move async
            await fs.promises.rename( fromPath, toPath );

            // Log because we're crazy
            console.log( "Moved '%s'->'%s'", fromPath, toPath );
        } // End for...of
    }
    catch( e ) {
        // Catch anything bad that happens
        console.error( "We've thrown! Whoops!", e );
    }

})(); // Wrap in parenthesis and call now