Loading and displaying large text files

Question

In a Swing application, I sometimes need to support read-only access to large, line-oriented text files that are slow to load: logs, dumps, traces, etc. For small amounts of

Answer 1

As I was struggeling with a similar use case I implemented a simple paging solution. It is far from perfect but works maybe someone finds it helpful.

In combination with a jtextarea it works ok but with a JEditorPane the performance is miserable.

If someone comes up with a better solution I would like to know about.

package net.ifao.tools.arcticrequester.gui.panel;


import java.awt.Adjustable;
import java.awt.event.AdjustmentEvent;
import java.awt.event.AdjustmentListener;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.StringReader;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;

import javax.swing.JEditorPane;
import javax.swing.JScrollPane;
import javax.swing.text.Document;
import javax.swing.text.EditorKit;
import javax.swing.text.JTextComponent;


/**
 * A class that manages the visibility of file content visualized with a textarea within a scrollbar.
 * Approx. 2050 lines from the file are visible at a time. Data is loaded from a file and
 * displayed while the user is scrolling. The chunks are loaded dynamically.
 *
 * @author dostricki
 *
 */
public class VisibilityManager
   implements AdjustmentListener
{

   private int lastLoadedLineFrom;
   private int lastLoadedLineTo;
   private int numberOfLines = 0;
   private File file;

   private boolean enabled = false;
   private boolean showLines = false;

   // load 1000 lines before the first visible line
   // and 1000 lines after the last vissible line
   private static final int LOAD_LINES_BEFORE_AND_AFTER_VIEWPORT = 1000;

   // margin until when no load is triggered.
   // moving the viewport more then 900 lines up or down should trigger a reload
   private static final int VIEWPORT_LINES_MOVE_THRASHOLD = 900;


   private JScrollPane scrollPane;
   private JTextComponent textComponent;

   private final BlockingQueue queue;

   public VisibilityManager(JScrollPane scrollPane, JTextComponent textArea)
   {
      this.scrollPane = scrollPane;
      this.textComponent = textArea;
      queue = new LinkedBlockingDeque<>();
      startConsumer();

      scrollPane.getVerticalScrollBar().addAdjustmentListener(this);
   }

   private void startConsumer()
   {
      Thread scrollEventConsumer = new Thread()
      {

         @Override
         public void run()
         {
            while (true) {
               try {

                  // if multiple events occured just process one
                  queue.take();
                  if (!queue.isEmpty()) {
                     List events = new ArrayList<>();
                     queue.drainTo(events);
                     //System.out.println("Handling scroll event. " + events.size() + " queued events dropped");
                  }

                  doHandleScrollEvent();

               }
               catch (InterruptedException e) {
                  e.printStackTrace();
               }
            }
         }
      };
      scrollEventConsumer.start();
   }

   public void setFile(File file)
   {
      this.file = file;

      try {
         this.numberOfLines = countNumberOfLines(file);
      }
      catch (IOException e1) {
         e1.printStackTrace();
      }
      int showLineMax = Math.min(getNumberOfLines(), 100);

      // show the first chunk immediately
      showLinesBuffererdReader(1, showLineMax, 0);

      this.enabled = true;
   }

   /**
    * precalculates the number of lines in the document - necessary
    * to replace the correct amount of preceeding and following
    * lines with EOL's so that the height of the scrollpane does never change.
    *
    * @param file
    * @return
    * @throws IOException
    */
   private int countNumberOfLines(File file)
      throws IOException
   {

      int numberOfLines = 0;

      //@formatter:off
      try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file),StandardCharsets.UTF_8));) {
         while (reader.ready()) {
            reader.readLine();
            ++numberOfLines;
         }
      }
      //@formatter:on

      return numberOfLines;
   }


   /****************************************
    *                Getter
    ****************************************/


   public int getNumberOfLines()
   {
      return numberOfLines;
   }

   public int getNumberOfLinesBuffer()
   {
      return LOAD_LINES_BEFORE_AND_AFTER_VIEWPORT;
   }

   public boolean isEnabled()
   {
      return enabled;
   }

   /****************************************
    *              Setter
    ****************************************/


   public void setLastLoadedLines(int lineFrom, int lineTo)
   {
      this.lastLoadedLineFrom = lineFrom;
      this.lastLoadedLineTo = lineTo;
   }

   public void setEnabled(boolean enabled)
   {
      this.enabled = enabled;
   }

   public void setShowLines(boolean showLines)
   {
      this.showLines = showLines;
   }


   /****************************************
    *           Calculation
    ****************************************/


   private boolean needsUpdate(int fromLine, int toLine)
   {
      boolean isBefore = fromLine < (this.lastLoadedLineFrom - VIEWPORT_LINES_MOVE_THRASHOLD);
      boolean isAfter = toLine > (this.lastLoadedLineTo + VIEWPORT_LINES_MOVE_THRASHOLD);

      if (isBefore || isAfter) {
         return true;
      } else {
         return false;
      }
   }

   private void showLinesBuffererdReader(int from, int to, int firstLineVisible)
   {
      //load also the buffer lines before
      from = from - getNumberOfLinesBuffer();
      //make sure it's valid
      from = Math.max(1, from);

      // load also the buffer lines after
      to = to + getNumberOfLinesBuffer();
      //make sure it's valid
      to = Math.min(getNumberOfLines(), to);

      FileChannel fileChannel = null;
      try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)))) {

         StringBuffer content = new StringBuffer();
         int newCaretPosition = 0;

         // fill leading empty lines
         for (long i = 1; i < from; ++i) {
            if (i == firstLineVisible) {
               newCaretPosition = content.length() + 1;
            }
            if (showLines) {
               content.append(i).append(": ");
            }
            content.append('\n');
         }

         // read/write lines with content
         int j = 0;
         while (reader.ready() && j <= to) {
            ++j;
            String line = reader.readLine();
            if (j >= from && j <= to) {
               if (j == firstLineVisible) {
                  newCaretPosition = content.length() + 1;
               }
               if (showLines) {
                  content.append(j).append(": ");
               }
               content.append(line).append('\n');
            }
         }

         // fill trailing empty lines
         for (int i = to + 1; i <= getNumberOfLines(); ++i) {
            if (i == firstLineVisible) {
               newCaretPosition = content.length() + 1;
            }
            if (showLines) {
               content.append(i).append(": ");
            }
            content.append('\n');
         }

         updateTextInUI(content);

         // workaround for page up/down - it changes the caret position
         // so we are re-setting it to the first visible line
         // scrolling by scrollbars does not change the caret
         //textComponent.setCaretPosition(newCaretPosition);
      }
      catch (IOException e) {
         e.printStackTrace();
      }
      finally {

         try {
            if (fileChannel != null) {
               fileChannel.close();
            }
         }
         catch (IOException e) {
            e.printStackTrace();
         }
      }
   }

   /**
    * @param content
    * @throws IOException
    */
   private void updateTextInUI(StringBuffer content)
      throws IOException
   {
      if (textComponent instanceof JEditorPane) {
         JEditorPane edit = ((JEditorPane) textComponent);
         EditorKit editorKit = edit.getEditorKit();
         Document createDefaultDocument = editorKit.createDefaultDocument();
         createDefaultDocument.putProperty("IgnoreCharsetDirective", Boolean.TRUE);
         try {
            editorKit.read(new StringReader(content.toString()), createDefaultDocument, 0);
         }
         catch (Exception e) {
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            e.printStackTrace(new PrintStream(out));
            edit.setText(new String(out.toByteArray()));
         }
         edit.setDocument(createDefaultDocument);
      } else {
         textComponent.setText(content.toString());
      }
   }


   /****************************************
    *           Eventing
    ****************************************/


   /**
    * fired when scrolling happens in any of the cases and ways.
    * Events are cached through a queue so that simultanious events
    * don't trigger unnecessary update actions
    * @see java.awt.event.AdjustmentListener#adjustmentValueChanged(java.awt.event.AdjustmentEvent)
    */
   @Override
   public void adjustmentValueChanged(AdjustmentEvent evt)
   {
      Adjustable source = evt.getAdjustable();
      if (evt.getValueIsAdjusting()) {
         return;
      }

      if (source != null) {
         try {
            queue.put(source);
         }
         catch (InterruptedException e) {
            e.printStackTrace();
         }
      }
   }


   private void doHandleScrollEvent()
   {
      // determine which lines to request to be loaded into the
      int height = this.scrollPane.getVerticalScrollBar().getMaximum();
      int lines = getNumberOfLines();
      if (lines == 0) {
         return;
      }
      float heightPerLine = height / lines;
      int visibleLines = Math.round(this.scrollPane.getVerticalScrollBar().getVisibleAmount() / heightPerLine);
      int firstLineVisible = (int) Math.ceil(this.scrollPane.getVerticalScrollBar().getValue() / heightPerLine);

      int fromLine = Math.max(firstLineVisible, 1);
      if (fromLine > lines) {
         fromLine = lines;
      }

      int toLine = Math.min(firstLineVisible + visibleLines, lines);
      if (needsUpdate(fromLine, toLine)) {
         if (enabled) {
            setLastLoadedLines(fromLine, toLine);
            showLinesBuffererdReader(fromLine, toLine, firstLineVisible);
         }
      }
   }
}

usage:

import java.awt.BorderLayout;
import java.awt.EventQueue;
import java.awt.event.MouseEvent;
import java.awt.event.MouseListener;
import java.io.File;

import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
import javax.swing.UIManager;
import javax.swing.UnsupportedLookAndFeelException;
import javax.swing.border.EmptyBorder;
import javax.swing.text.DefaultCaret;

import net.ifao.tools.arcticrequester.gui.panel.VisibilityManager;


public class TestFrame
   extends JFrame
   implements MouseListener
{

   private VisibilityManager visibilityManager;


   public static void main(String[] args)
   {
      EventQueue.invokeLater(new Runnable()
      {
         @Override
         public void run()
         {
            try {
               TestFrame frame = new TestFrame();
               frame.setVisible(true);
            }
            catch (Exception e) {
               e.printStackTrace();
            }
         }
      });
   }

   /**
    * Create the frame.
    */
   public TestFrame()
   {
      try {
         UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
      }
      catch (ClassNotFoundException | InstantiationException | IllegalAccessException | UnsupportedLookAndFeelException e2) {
         // TODO Auto-generated catch block
         e2.printStackTrace();
      }

      setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
      setBounds(100, 100, 650, 500);

      JPanel contentPane = new JPanel();
      contentPane.setBorder(new EmptyBorder(5, 5, 5, 5));
      setContentPane(contentPane);
      contentPane.setLayout(new BorderLayout(0, 0));

      JTextArea textArea = new JTextArea();
      textArea.setEditable(false);
      textArea.addMouseListener(this);
      textArea.setAutoscrolls(false);

      textArea.setCaretPosition(0);
      DefaultCaret caret = (DefaultCaret) textArea.getCaret();
      caret.setUpdatePolicy(DefaultCaret.NEVER_UPDATE);

      JScrollPane scrollPane = new JScrollPane(textArea);

      contentPane.add(scrollPane);

      visibilityManager = new VisibilityManager(scrollPane, textArea);

      visibilityManager.setShowLines(true);
      File file = new File("C:/iFAO/workspaces/polaris2/git/requester/ArcticRequester/src/test/java/responseview_20200603.tmp");
      visibilityManager.setFile(file);


      this.dispose();
   }


   /**
    * @see java.awt.event.MouseListener#mouseClicked(java.awt.event.MouseEvent)
    */
   @Override
   public void mouseClicked(MouseEvent e)
   {
      boolean doScroll = !visibilityManager.isEnabled();
      this.visibilityManager.setEnabled(doScroll);
      System.out.println("scrolling set to " + doScroll);
   }

   /**
    * @see java.awt.event.MouseListener#mousePressed(java.awt.event.MouseEvent)
    */
   @Override
   public void mousePressed(MouseEvent e)
   {
      // TODO Auto-generated method stub

   }

   /**
    * @see java.awt.event.MouseListener#mouseReleased(java.awt.event.MouseEvent)
    */
   @Override
   public void mouseReleased(MouseEvent e)
   {
      // TODO Auto-generated method stub

   }

   /**
    * @see java.awt.event.MouseListener#mouseEntered(java.awt.event.MouseEvent)
    */
   @Override
   public void mouseEntered(MouseEvent e)
   {
      // TODO Auto-generated method stub

   }

   /**
    * @see java.awt.event.MouseListener#mouseExited(java.awt.event.MouseEvent)
    */
   @Override
   public void mouseExited(MouseEvent e)
   {
      // TODO Auto-generated method stub

   }

}