What is the fastest way to read a large number of small files into memory?

Question

I need to read ~50 files on every server start and place each text file\'s representation into memory. Each text file will have its own string (which is the best type to use

Answer 1

After searching across google for for existing tests on IO speed in Java, I must say TofuBear's test case completely opened my eyes. You have to run his test on your own platform to see what is fastest for you.

After running his test, and adding a few of my own (Credit to TofuBear for posting his original code), it appears you may get even more speed by using your own custom buffer vs. using the BufferedInputStream.

To my dismay the NIO ByteBuffer did not perform well.

NOTE: The static byte[] buffer shaved off a few ms, but the static ByteBuffers actualy increased the time to process! Is there anything wrong with the code??

I added a few tests:

1. ArrayTest_CustomBuffering (Read data directly into my own buffer)

2. ArrayTest_CustomBuffering_StaticBuffer (Read Data into a static buffer that is created only once in the beginning)

3. FileChannelArrayByteBuffer (use NIO ByteBuffer and wrapping your own byte[] array)

4. FileChannelAllocateByteBuffer (use NIO ByteBuffer with .allocate)

5. FileChannelAllocateByteBuffer_StaticBuffer (same as 4 but with a static buffer)

6. FileChannelAllocateDirectByteBuffer (use NIO ByteBuffer with .allocateDirect)

7. FileChannelAllocateDirectByteBuffer_StaticBuffer (same as 6 but with a static buffer)

Here are my results:, using Windows Vista and jdk1.6.0_13 on the extracted rt.jar: ArrayTest
time = 2075
bytes = 2120336424
ArrayTest
time = 2044
bytes = 2120336424
ArrayTest_CustomBuffering
time = 1903
bytes = 2120336424
ArrayTest_CustomBuffering_StaticBuffer
time = 1872
bytes = 2120336424
DataInputByteAtATime
time = 2668
bytes = 2120336424
DataInputReadFully
time = 2028
bytes = 2120336424
MemoryMapped
time = 2901
bytes = 2120336424
FileChannelArrayByteBuffer
time = 2371
bytes = 2120336424
FileChannelAllocateByteBuffer
time = 2356
bytes = 2120336424
FileChannelAllocateByteBuffer_StaticBuffer
time = 2668
bytes = 2120336424
FileChannelAllocateDirectByteBuffer
time = 2512
bytes = 2120336424
FileChannelAllocateDirectByteBuffer_StaticBuffer
time = 2590
bytes = 2120336424

My hacked version of TofuBear's code:

import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.MappedByteBuffer;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.util.HashSet;
import java.util.Set;
public class Main { 
    public static void main(final String[] argv)     { 
        ArrayTest.mainx(argv);
        ArrayTest.mainx(argv);
        ArrayTest_CustomBuffering.mainx(argv);
        ArrayTest_CustomBuffering_StaticBuffer.mainx(argv);
        DataInputByteAtATime.mainx(argv);
        DataInputReadFully.mainx(argv);
        MemoryMapped.mainx(argv);
        FileChannelArrayByteBuffer.mainx(argv);
        FileChannelAllocateByteBuffer.mainx(argv);
        FileChannelAllocateByteBuffer_StaticBuffer.mainx(argv);
        FileChannelAllocateDirectByteBuffer.mainx(argv);
        FileChannelAllocateDirectByteBuffer_StaticBuffer.mainx(argv);
     } 
 } 
abstract class Test { 
    static final int BUFF_SIZE = 20971520;
    static final byte[] StaticData = new byte[BUFF_SIZE];
    static final ByteBuffer StaticBuffer =ByteBuffer.allocate(BUFF_SIZE);
    static final ByteBuffer StaticDirectBuffer = ByteBuffer.allocateDirect(BUFF_SIZE);
    public final void run(final File root)     { 
        final Set files;
        final long      size;
        final long      start;
        final long      end;
        final long      total;
        files = new HashSet();
        getFiles(root, files);
        start = System.currentTimeMillis();
        size = readFiles(files);
        end = System.currentTimeMillis();
        total = end - start;
        System.out.println(getClass().getName());
        System.out.println("time  = " + total);
        System.out.println("bytes = " + size);
     } 
    private void getFiles(final File dir,final Set files)     { 
        final File[] childeren;
        childeren = dir.listFiles();
        for(final File child : childeren)         { 
            if(child.isFile())             { 
                files.add(child);
             } 
            else             { 
                getFiles(child, files);
             } 
         } 
     } 
    private long readFiles(final Set files)     { 
        long size;
        size = 0;
        for(final File file : files)         { 
            size += readFile(file);
         } 
        return (size);
     } 
    protected abstract long readFile(File file);
 } 
class ArrayTest    extends Test { 
    public static void mainx(final String[] argv)     { 
        final Test test;
        test = new ArrayTest();
        test.run(new File(argv[0]));
     } 
    protected long readFile(final File file)     { 
        InputStream stream;
        stream = null;
        try         { 
            final byte[] data;
            int          soFar;
            int          sum;
            stream = new BufferedInputStream(new FileInputStream(file));
            data   = new byte[(int)file.length()];
            soFar  = 0;
            do             { 
                soFar += stream.read(data, soFar, data.length - soFar);
             } 
            while(soFar != data.length);
            sum = 0;
            for(final byte b : data)             { 
                sum += b;
             } 
            return (sum);
         } 
        catch(final IOException ex)         { 
            ex.printStackTrace();
         } 
        finally         { 
            if(stream != null)             { 
                try                 { 
                    stream.close();
                 } 
                catch(final IOException ex)                 { 
                    ex.printStackTrace();
                 } 
             } 
         } 
        return (0);
     } 
 } 

 class ArrayTest_CustomBuffering    extends Test { 
    public static void mainx(final String[] argv)     { 
        final Test test;
        test = new ArrayTest_CustomBuffering();
        test.run(new File(argv[0]));
     } 
    protected long readFile(final File file)     { 
        InputStream stream;
        stream = null;
        try         { 
            final byte[] data;
            int          soFar;
            int          sum;
            stream = new FileInputStream(file);
            data   = new byte[(int)file.length()];
            soFar  = 0;
            do             { 
                soFar += stream.read(data, soFar, data.length - soFar);
             } 
            while(soFar != data.length);
            sum = 0;
            for(final byte b : data)             { 
                sum += b;
             } 
            return (sum);
         } 
        catch(final IOException ex)         { 
            ex.printStackTrace();
         } 
        finally         { 
            if(stream != null)             { 
                try                 { 
                    stream.close();
                 } 
                catch(final IOException ex)                 { 
                    ex.printStackTrace();
                 } 
             } 
         } 
        return (0);
     } 
 }

 class ArrayTest_CustomBuffering_StaticBuffer    extends Test { 



    public static void mainx(final String[] argv)     { 
        final Test test;
        test = new ArrayTest_CustomBuffering_StaticBuffer();
        test.run(new File(argv[0]));
     } 
    protected long readFile(final File file)     { 
        InputStream stream;
        stream = null;
        try         { 
            int          soFar;
            int          sum;
            final int    fileSize;
            stream = new FileInputStream(file);
            fileSize = (int)file.length();
            soFar  = 0;
            do             { 
                soFar += stream.read(StaticData, soFar, fileSize - soFar);
             } 
            while(soFar != fileSize);
            sum = 0;
            for(int i=0;i