Compact (archive) old log files in python

Question

I'm using standart logger library in Python. There are RotatingFileHandler, that can rotate log files dayily, for example.

But it just renames them. Will be great, if it can not only rename, but also put old files in zip (or gz, bzip, etc) archive.

Is there easy way to achieve this?

Answer 1

I think your best option is to extend RotatingFileHandler something like this (not tested):

import os
from logging.handlers import RotatingFileHandler


COMPRESSION_SUPPORTED = {}

try:
   import gzip
   COMPRESSION_SUPPORTED['gz'] = gzip
except ImportError:
   pass

try:
   import zipfile
   COMPRESSION_SUPPORTED['zip'] = zipfile
except ImportError:
   pass


class NewRotatingFileHandler(RotatingFileHandler):

     def __init__(self, *args, **kws):
         compress_mode = kws.pop('compress_mode')

         try:
             self.compress_cls = COMPRESSION_SUPPORTED[compress_mode]
         except KeyError:
             raise ValueError('"%s" compression method not supported.' % compress_mode)

         super(NewRotatingFileHandler, self).__init__(self, *args, **kws)

     def doRollover(self):
         super(NewRotatingFileHandler, self).doRollover()

         # Compress the old log.
         old_log = self.baseFilename + ".1"
         with open(old_log) as log:
             with self.compress_cls.open(old_log + '.gz', 'wb') as comp_log:
                 comp_log.writelines(log)

         os.remove(old_log)

Answer 2

The accepted answer will archive only 1 file-(basefile.log.1) .The other files are not archived. This code will archive all the log files other than the base file.

import os
import gzip
import logging.handlers

class NewRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def __init__(self, filename, **kws):
        backupCount = kws.get('backupCount', 0)
        self.backup_count = backupCount
        logging.handlers.RotatingFileHandler.__init__(self, filename, **kws)

    def doArchive(self, old_log):
        with open(old_log) as log:
            with gzip.open(old_log + '.gz', 'wb') as comp_log:
                comp_log.writelines(log)
       os.remove(old_log)

   def doRollover(self):
      if self.stream:
          self.stream.close()
          self.stream = None
      if self.backup_count > 0:
          for i in range(self.backup_count - 1, 0, -1):
              sfn = "%s.%d.gz" % (self.baseFilename, i)
              dfn = "%s.%d.gz" % (self.baseFilename, i + 1)
              if os.path.exists(sfn):
                  if os.path.exists(dfn):
                      os.remove(dfn)
                  os.rename(sfn, dfn)
      dfn = self.baseFilename + ".1"
      if os.path.exists(dfn):
          os.remove(dfn)
      if os.path.exists(self.baseFilename):
          os.rename(self.baseFilename, dfn)
          self.doArchive(dfn)
      if not self.delay:
          self.stream = self._open()

Answer 3

You can automatically write bz2 compressed log files by initializing the RotatingFileHandler with encoding='bz2-codec':

import logging
import logging.handlers as handlers

if __name__=='__main__':
    log_filename='log_rotate.bz2'
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    handler = handlers.RotatingFileHandler(
        log_filename, maxBytes=20, backupCount=5, encoding='bz2-codec')
    logger.addHandler(handler)
    for i in range(20):
        logger.debug('i = %d' % i)

PS. Python3 removed 'bz2-codec' from the set of valid encodings, so this solution is specific to Python2.