How to create Celery Windows Service?

Question

I\'m trying to create a Windows Service to launch Celery. I have come across an article that does it using Task Scheduler. However it seems to launch numerous celery instanc

Answer 1

Thanks to Azalea as this led me to the path in being able to create 2 windows services with Celery 4 on Windows.

One to be able to start/stop multiple workers T Second to be able to start/stop the beat service and tidy up the pid using Celery 4.

Only caveat in this that I do not have a solution for is you can not restart the workers as you need to ensure the spawned processes for multiple are stopped before starting back up.

Workers.py:

'''Usage : python celery_service.py install (start / stop / remove)
Run celery as a Windows service
'''
import win32service
import win32serviceutil
import win32api
import win32con
import win32event
import subprocess
import sys
import os
import shlex
import logging
import time

# The directory for celery_worker.log and celery_worker_service.log
# Default: the directory of this script
INSTDIR = 'X:\Application\Project'
LOGDIR = 'X:\Application\LogFiles'
# The path of python Scripts
# Usually it is in PYTHON_INSTALL_DIR/Scripts. e.g.
# r'C:\Python27\Scripts'
# If it is already in system PATH, then it can be set as ''
PYTHONSCRIPTPATH = 'C:\Python36\Scripts'
# The directory name of django project
# Note: it is the directory at the same level of manage.py
# not the parent directory
PROJECTDIR = 'Project'

logging.basicConfig(
    filename = os.path.join(LOGDIR, 'celery_worker_service.log'),
    level = logging.DEBUG, 
    format = '[%(asctime)-15s: %(levelname)-7.7s] %(message)s'
)

class CeleryService(win32serviceutil.ServiceFramework):

    _svc_name_ = "CeleryWorkers"
    _svc_display_name_ = "CeleryWorkers"

    def __init__(self, args): 
        win32serviceutil.ServiceFramework.__init__(self, args)
        self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)

    def SvcStop(self):
        logging.info('Stopping {name} service ...'.format(name=self._svc_name_))        
        self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
        win32event.SetEvent(self.hWaitStop)
        logging.info('Stopped1 {name} service ...'.format(name=self._svc_name_))   
        logging.info('Stopped3 {name} service ...'.format(name=self._svc_name_)) 
        command = '"{celery_path}" -A {proj_dir} --workdir=X:/Application/Project control shutdown --timeout=10'.format(
        celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
        proj_dir=PROJECTDIR,
        log_path=os.path.join(LOGDIR,'celery_worker.log'))
        logging.info('command: ' + command)
        args = shlex.split(command)
        proc = subprocess.Popen(args)
        logging.info('Stopped celery shutdown  ...') 
        self.ReportServiceStatus(win32service.SERVICE_STOPPED)
        logging.info('Stopped2 {name} service ...'.format(name=self._svc_name_))  
        sys.exit()           

    def SvcDoRun(self):
        logging.info('Starting {name} service ...'.format(name=self._svc_name_))
        os.chdir(INSTDIR) # so that proj worker can be found
        logging.info('cwd: ' + os.getcwd())
        self.ReportServiceStatus(win32service.SERVICE_RUNNING)
        command = '"{celery_path}" -A {proj_dir} -c 8 worker --workdir=X:/Application/Project --pidfile=celeryservice.pid  -f "{log_path}" -l info'.format(
            celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
            proj_dir=PROJECTDIR,
            log_path=os.path.join(LOGDIR,'celery_worker.log'))
        logging.info('command: ' + command)
        args = shlex.split(command)
        proc = subprocess.Popen(args)
        logging.info('pid: {pid}'.format(pid=proc.pid))
        self.timeout = 3000
        while True:
            rc = win32event.WaitForSingleObject(self.hWaitStop, self.timeout)
            if rc == win32event.WAIT_OBJECT_0:
                # stop signal encountered
                # terminate process 'proc'
                PROCESS_TERMINATE = 1
                handle = win32api.OpenProcess(PROCESS_TERMINATE, False, proc.pid)
                win32api.TerminateProcess(handle, -1)
                win32api.CloseHandle(handle)                
                break

if __name__ == '__main__':
   win32serviceutil.HandleCommandLine(CeleryService)

Beatservice.py:

'''Usage : python celery_service.py install (start / stop / remove)
Run celery as a Windows service
'''
import win32service
import win32serviceutil
import win32api
import win32con
import win32event
import subprocess
import sys
import os
import shlex
import logging
import time
import signal

# The directory for celery_beat.log and celery_beat_service.log
# Default: the directory of this script
INSTDIR = os.path.dirname(os.path.realpath(__file__))
LOGPATH = 'X:\Application\Logs'
# The path of python Scripts
# Usually it is in PYTHON_INSTALL_DIR/Scripts. e.g.
# r'C:\Python27\Scripts'
# If it is already in system PATH, then it can be set as ''
PYTHONSCRIPTPATH = 'C:\Python36\Scripts'
# The directory name of django project
# Note: it is the directory at the same level of manage.py
# not the parent directory
PROJECTDIR = 'PROJECT'

logging.basicConfig(
    filename = os.path.join(LOGPATH, 'celery_beat_service.log'),
    level = logging.DEBUG, 
    format = '[%(asctime)-15s: %(levelname)-7.7s] %(message)s'
)

class CeleryService(win32serviceutil.ServiceFramework):

    _svc_name_ = "CeleryBeat"
    _svc_display_name_ = "CeleryBeat"

    def __init__(self, args):
        win32serviceutil.ServiceFramework.__init__(self, args)
        self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)           

    def SvcStop(self):
        logging.info('Stopping 1 {name} service ...'.format(name=self._svc_name_))        
        self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
        win32event.SetEvent(self.hWaitStop)
        pidno = open("X:\Aplication\Project\celerybeat.pid", "r")
        _pid_id_ = pidid=pidno.read()
        pidno.close()
        logging.info(_pid_id_)
        logging.info('taskkill /F /PID {pidid} ..'.format(pidid=_pid_id_))
        cmdcom = 'taskkill /F /PID {pidid}'.format(pidid=_pid_id_)
        logging.info(cmdcom)
        killargs = shlex.split(cmdcom)
        process = subprocess.Popen(killargs)
        output, error = process.communicate()
        logging.info(output)
        logging.info('Stopping 2 {name} service ...'.format(name=self._svc_name_))
        os.remove("X:\Application\PROJECT\celerybeat.pid")
        logging.info('X:\Application\PROJECT\celerybeat.pid  file removed')
        self.ReportServiceStatus(win32service.SERVICE_STOPPED)
        sys.exit()

    def SvcDoRun(self):
        logging.info('Starting {name} service ...'.format(name=self._svc_name_))
        os.chdir(INSTDIR) # so that proj worker can be found
        logging.info('cwd: ' + os.getcwd())
        self.ReportServiceStatus(win32service.SERVICE_RUNNING)
        command = '"{celery_path}" -A {proj_dir} beat --workdir=X:/Application/Project -f X:/Application/logs/beat.log -l info'.format(
            celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
            proj_dir=PROJECTDIR,
            log_path=os.path.join(LOGPATH,'celery_beat.log'))
        logging.info('command: ' + command)
        args = shlex.split(command)
        proc = subprocess.Popen(args)
        logging.info('pid: {pid}'.format(pid=proc.pid))
        self.timeout = 3000
        while True:
            rc = win32event.WaitForSingleObject(self.hWaitStop, self.timeout)
            if rc == win32event.WAIT_OBJECT_0:
                # stop signal encountered
                # terminate process 'proc'
                PROCESS_TERMINATE = 1
                handle = win32api.OpenProcess(PROCESS_TERMINATE, False, proc.pid)
                win32api.TerminateProcess(handle, -1)
                win32api.CloseHandle(handle)                
                break

if __name__ == '__main__':
   win32serviceutil.HandleCommandLine(CeleryService)