yunfei_monitor/monitor/supervisord/yf_supervisord.py

# coding:utf-8

import os
import sys
import fs
import psutil
import subprocess
from crontab import CronTab

local_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if local_path not in sys.path:
    sys.path.append(local_path)

from utils.comm_tools import shell_cmd, get_monitor_config


class SupervisorTools:
    def __init__(self):
        supervisor_dir = "/data/supervisor"
        self.base_config_file = "/etc/supervisor/supervisord.conf"

        if supervisor_dir[-1] == '/':
            supervisor_dir = supervisor_dir[: -1]
        temp_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        utils_dir = os.path.join(temp_dir, 'utils')
        self.exception_warning_file = os.path.join(utils_dir, 'exception_warning.py')
        self.config_base = supervisor_dir

        self.python_bin = sys.executable
        self.bin_dir = os.path.dirname(self.python_bin)
        self.supervisord = os.path.join(self.bin_dir, 'supervisord')
        self.supervisorctl = os.path.join(self.bin_dir, 'supervisorctl')
        self.config_dir = os.path.join(self.config_base, 'config.d')
        self.log_dir = os.path.join(self.config_base, 'log')

        tmp_dir = os.path.dirname(self.base_config_file)
        if not os.path.isdir(tmp_dir):
            os.makedirs(tmp_dir)
        if not os.path.isdir(self.config_dir):
            os.makedirs(self.config_dir)
        if not os.path.isdir(self.log_dir):
            os.makedirs(self.log_dir)

    def _get_supervisor_app(self):
        '''获取supervisor所管理的程序'''
        status, app_name_list = False, []
        cmd = '{} status'.format(self.supervisorctl)
        code, stdout, stderr = shell_cmd(cmd)

        if code >= 0:
            for line in stdout.splitlines():
                tmp_line = line.split()
                if len(tmp_line) > 1:
                    app_name = tmp_line[0].strip()
                    app_name_list.append(app_name)
            status, msg = True, set(app_name_list)
        else:
            msg = '获取supervisor所管理的程序失败: {}'.format(stderr)
        return status, msg

    def _make_task_conf(self, app_name, dir_name, cmd, user):
        '''生成任务配置文件'''
        log_dir = os.path.join(self.log_dir, app_name)
        if not os.path.isdir(log_dir):
            os.makedirs(log_dir)
        conf_path = os.path.join(self.config_dir, '{}.conf'.format(app_name))
        log_file = os.path.join(log_dir, '{}.log'.format(app_name))

        conf_str = '''
#程序唯一名称
[program:{app_name}]

#程序路径                                       
directory={dir_name}

#运行程序的命令                                       
command={cmd}

#标准日志输出位置
stdout_logfile={log_file}

autostart=true                          #supervisord启动后，同时启动此程序                                  
startsecs=5                             #启动5秒后没有异常退出，就表示进程正常启动了，默认为1秒
autorestart=true                        #程序退出后自动重启
startretries=3                          #启动失败自动重试次数，默认是3                   
user={user}                               #用哪个用户启动进程，默认是root                                      
redirect_stderr=true                    #把stderr重定向到stdout标准输出，默认false            
stdout_logfile_maxbytes=50MB            #stdout标准输出日志文件大小,日志文件备份数
stdout_logfile_backups=10
stopasgroup=true                        # 停止或终止进程时，同时结束进程包含的子进程 
killasgroup=true

'''.format(app_name=app_name, dir_name=dir_name, cmd=cmd, log_file=log_file, user=user)
        with open(conf_path, 'w') as fw:
            fw.write(conf_str)

    def _make_supervisor_conf(self):
        cmd = '{} -u {}'.format(self.python_bin, self.exception_warning_file)
        conf = '''
[unix_http_server]
file={supervisor_dir}/supervisor.sock   ; the path to the socket file
;chmod=0700                 ; socket file mode (default 0700)
;chown=nobody:nogroup       ; socket file uid:gid owner
;username=user              ; default is no username (open server)
;password=123               ; default is no password (open server)

[inet_http_server]         ; inet (TCP) server disabled by default
port=127.0.0.1:59001        ; ip_address:port specifier, *:port for all iface
username=yunfei_yanfabu              ; default is no username (open server)
password=yanfabu6021               ; default is no password (open server)

[supervisord]
logfile={supervisor_dir}/supervisord.log ; main log file; default $CWD/supervisord.log
logfile_maxbytes=50MB        ; max main logfile bytes b4 rotation; default 50MB
logfile_backups=10           ; # of main logfile backups; 0 means none, default 10
loglevel=info                ; log level; default info; others: debug,warn,trace
pidfile={supervisor_dir}/supervisord.pid ; supervisord pidfile; default supervisord.pid
nodaemon=false               ; start in foreground if true; default false
silent=false                 ; no logs to stdout if true; default false
minfds=1024                  ; min. avail startup file descriptors; default 1024
minprocs=200                 ; min. avail process descriptors;default 200
;umask=022                   ; process file creation umask; default 022
user=root                    ; setuid to this UNIX account at startup; recommended if root
;identifier=supervisor       ; supervisord identifier, default is 'supervisor'
;directory=/tmp              ; default is not to cd during start
;nocleanup=true              ; don't clean up tempfiles at start; default false
;childlogdir=/tmp            ; 'AUTO' child log dir, default $TEMP
;environment=KEY="value"     ; key value pairs to add to environment
;strip_ansi=false            ; strip ansi escape codes in logs; def. false

[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix://{supervisor_dir}/supervisor.sock
;username=chris              ; should be same as in [*_http_server] if set
;password=123                ; should be same as in [*_http_server] if set
;prompt=mysupervisor         ; cmd line prompt (default "supervisor")
;history_file=~/.sc_history  ; use readline history if available

[eventlistener:crash_warning]
command={exception_warning_file}
events=PROCESS_STATE
redirect_stderr=false

[include]
files = {supervisor_dir}/config.d/*.conf

'''.format(exception_warning_file=cmd, supervisor_dir=self.config_base)
        print(cmd)
        with open(self.base_config_file, 'w') as fw:
            fw.write(conf)

    def _update_task(self):
        '''启动添加的程序'''
        status, msg = True, 'supervisor管理器启动程序成功'
        cmd = '{} update all'.format(self.supervisorctl)
        code, stdout, stderr = shell_cmd(cmd)
        if code != 0:
            status, msg = False, 'supervisor管理器启动程序失败:{}\n{}'.format(stdout, stderr)
        return status, msg

    def make_task(self):
        '''创建supervisor任务并启动'''
        status, msg = self._get_supervisor_app()
        if not status:
            return status, msg
        app_name_set, app_name_list = msg, []
        monitor_config = get_monitor_config()
        with fs.open_fs('/') as root_fs:
            for script_dir, item in monitor_config.items():
                if not os.path.isdir(script_dir):
                    continue
                file_filter, file_exclude = item['filter'], item.get('exclude', [])
                python_bin, env_name, cmd_user = item['executable'], item['name'], item['user']
                if not file_exclude:
                    file_exclude = None
                if not file_filter:
                    file_filter = None
                for path in root_fs.walk.files(script_dir, filter=file_filter, exclude=file_exclude, max_depth=1):
                    dir_name, file_name = os.path.split(path)
                    index = file_name.rfind('.')
                    app_name = file_name[:index] if index != -1 else file_name
                    app_name = "{}_{}".format(env_name, app_name)
                    if app_name in app_name_set:
                        continue
                    cmd = '{} -u {}'.format(python_bin, path)
                    self._make_task_conf(app_name=app_name, dir_name=dir_name, cmd=cmd, user=cmd_user)
                    app_name_list.append(app_name)

        status, msg = True, 'supervisor管理器启动程序成功'
        if app_name_list:
            status, msg = self._update_task()
        return status, msg

    def _check_supervisord(self):
        '''	检测supervisor管理器进程是否运行'''
        is_run = False
        for proc in psutil.process_iter():
            proc_name = proc.name().lower()
            if proc_name == 'supervisord':
                is_run = True
                break
        return is_run

    def start_supervisord(self):
        '''启动supervisor管理器'''
        msg = ''
        if not os.path.isfile(self.supervisord):
            msg = 'supervisor未安装，请先安装supervisor'
        else:
            is_run = self._check_supervisord()
            if not is_run:
                self._make_supervisor_conf()
                cmd = "{} -c {}".format(self.supervisord, self.base_config_file)
                result = subprocess.run(cmd, shell=True)
                if result.returncode == 0:
                    is_run = True
                else:
                    msg = '启动supervisor管理器失败, 请检查'

            if is_run:
                status, msg = self.make_task()
        return msg


def _init_crond():
    cron = CronTab(user='root')
    python_bin = sys.executable
    main_file = os.path.abspath(__file__)
    cmd = "{} {}".format(python_bin, main_file)
    for job in cron:
        if job.command in cmd:
            break
    else:
        new_job = cron.new(cmd, comment="supervise监控脚本，禁止修改")
        new_job.minute.every(2)
        cron.write()


if __name__ == '__main__':
    _init_crond()

    st = SupervisorTools()
    msg = st.start_supervisord()
    print(msg)