Node.js & Amazon S3: How to iterate through all files in a bucket?

Question

Is there any Amazon S3 client library for Node.js that allows listing of all files in S3 bucket?

The most known aws2js and knox don\'t seem to have this functionalit

Answer 1

Using the new API s3.listObjectsV2 the recursive solution will be:

S3Dataset.prototype.listFiles = function(params,callback) {
    var self=this;

    var options = {
    };
    for (var attrname in params) { options[attrname] = params[attrname]; }

    var results=[];
    var s3=self.s3Store.GetInstance();
    function listAllKeys(token, callback) {
        var opt={ Bucket: self._options.s3.Bucket, Prefix: self._options.s3.Key, MaxKeys: 1000 };
        if(token) opt.ContinuationToken = token;
        s3.listObjectsV2(opt, (error, data) => {
            if (error) {
                if(self.logger) this.logger.error("listFiles error:", error);
                return callback(error);
            } else {
                for (var index in data.Contents) {
                    var bucket = data.Contents[index];
                    if(self.logger) self.logger.debug("listFiles Key: %s LastModified: %s Size: %s", bucket.Key, bucket.LastModified, bucket.Size);
                    if(bucket.Size>0) {
                        var Bucket=self._options.s3.Bucket;
                        var Key=bucket.Key;
                        var components=bucket.Key.split('/');
                        var name=components[components.length-1];
                        results.push({
                            name: name,
                            path: bucket.Key,
                            mtime: bucket.LastModified,
                            size: bucket.Size,
                            sizehr: formatSizeUnits(bucket.Size)
                        });
                    }
                }
                if( data.IsTruncated ) { // truncated page
                    return listAllKeys(data.NextContinuationToken, callback);
                } else {
                    return callback(null,results);
                }
            }
        });
    }
    return listAllKeys.apply(this,['',callback]);
};

where

function formatSizeUnits(bytes){
    if      (bytes>=1099511627776) {bytes=(bytes/1099511627776).toFixed(4)+' PB';}
    else if (bytes>=1073741824)    {bytes=(bytes/1073741824).toFixed(4)+' GB';}
    else if (bytes>=1048576)       {bytes=(bytes/1048576).toFixed(4)+' MB';}
    else if (bytes>=1024)          {bytes=(bytes/1024).toFixed(4)+' KB';}
    else if (bytes>1)              {bytes=bytes+' bytes';}
    else if (bytes==1)             {bytes=bytes+' byte';}
    else                           {bytes='0 byte';}
    return bytes;
}//formatSizeUnits