Python, want logging with log rotation and compression

Question

Can anyone suggest a way in python to do logging with:

• log rotation every day
• compression of logs when they\'re rotated
• optional - delete old

Answer 1

I have added below one solution where i am basically zipping old backup logs to zip with timestamp on it. using one extra variable called ZipbackupCount. # no of old files to be zipped

e.g. we have logs like this. (backupcount = 5 and ZipbackupCount = 2)

a.log.1

a.log.2

a.log.3

a.log.4

a.log.11-09-2020-11-11-11.zip

once count of backup logs hits to 5 it triggers to zip a.log.5 and a.log.4 to above zip and continues.

import os
import datetime
import gzip
import logging.handlers
import zipfile

from config.config import PROJECT_PATH, LOG_DIR, LOG_FILE_NAME, LOG_FILESIZE


class NewRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def __init__(self, filename, **kws):
        backupCount = kws.get('backupCount', 0)
        self.backup_count = backupCount
        self.ZipbackupCount = kws.pop('ZipbackupCount', 0)
        self.file_name = filename
        self.log_dir = os.path.split(self.file_name)[0]
        self.log_file_name = os.path.split(self.file_name)[-1]

        logging.handlers.RotatingFileHandler.__init__(self, filename, **kws)

    def doArchive(self, old_log):
        with open(old_log) as log:
            with gzip.open(old_log + '.gz', 'wb') as comp_log:
                comp_log.writelines(log)

        os.remove(old_log)

    def doRollover(self):
        super(NewRotatingFileHandler, self).doRollover()

        zip_file_name = self.log_file_name + "." + datetime.datetime.now().strftime("%d-%m-%Y-%H-%M-%S") + ".zip"

        if os.path.exists(self.rotation_filename("%s.%d" % (self.baseFilename, self.backupCount))) and self.ZipbackupCount > 0 and self.file_name:

            with zipfile.ZipFile(os.path.join(self.log_dir, zip_file_name), "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf:

                for i in range(self.backupCount, self.backupCount - self.ZipbackupCount, -1):

                    sfn = self.rotation_filename("%s.%d" % (self.baseFilename, i))

                    if os.path.exists(sfn):
                        zf.write(sfn, "%s.%d" % (self.log_file_name, i))
                        os.remove(sfn)
                    else:
                        continue

                zf.close()


# handler = NewRotatingFileHandler(filename=os.path.join(PROJECT_PATH, LOG_DIR, LOG_FILE_NAME),
#                                  maxBytes=LOG_FILESIZE, backupCount=5, ZipbackupCount=2)
#
# handler.doRollover()