yunfei_monitor/monitor/utils/exception_warning.py

# coding:utf-8

'''
异常警告事件监控脚本：supervisor程序异常中止时候自动发送异常警告消息到redis中
auth : zhaiyifei
time : 2021-12-14
'''


import os
import sys
import json
import datetime
from supervisor import childutils

local_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if local_path not in sys.path:
    sys.path.append(local_path)

from monitor.utils.comm_tools import get_redis_connection


def usage(exitstatus=255):
    sys.exit(exitstatus)


class CrashExceptionWarning:
    def __init__(self, programs, anyy):
        self.programs = programs
        self.any = anyy
        self.stdin = sys.stdin
        self.stdout = sys.stdout
        self.stderr = sys.stderr
        self.event_dict = {}
        self.con = get_redis_connection()

    def _get_now(self):
        '''获取时间'''
        now = datetime.datetime.now()
        now_str = now.strftime('%Y-%m-%d %H:%M:%S')
        return now, now_str

    def _clean_event(self):
        '''清理过期事件，超过30秒既为过期事件'''
        now = self._get_now()[0]
        key_list = []
        for key, value in self.event_dict.items():
            diff = now - value[0]
            if diff.seconds > 30:
                key_list.append(key_list)

        for key in key_list:
            if key in self.event_dict:
                self.event_dict.pop(key)

    def send_redis(self, content):
        '''推送消息redis'''
        data = json.dumps(content)
        self.con.rpush('yf_dingding', data)

    def runforever(self, test=False):
        while True:
            headers, payload = childutils.listener.wait(
                self.stdin, self.stdout)
            pheaders, pdata = childutils.eventdata(payload + '\n')

            app_name = pheaders['processname']
            if app_name == 'crasheding':
                childutils.listener.ok(self.stdout)
                continue

            eventname = headers['eventname']
            if eventname != 'PROCESS_STATE_EXITED':
                now_str = self._get_now()[1]
                if eventname == 'PROCESS_STATE_RUNNING':
                    if app_name in self.event_dict:
                        msg = '[{}] 程序于 {} 发生异常中止，并在 {} 自动重启 成功'.format(
                            app_name, self.event_dict[app_name], now_str)
                        self.event_dict.pop(app_name)
                        self.send_redis({'content': msg})
                elif eventname == 'PROCESS_STATE_FATAL':
                    msg = '[{}] 程序于 {} 启动失败'.format(app_name, now_str)
                    if app_name in self.event_dict:
                        msg = '[{}] 程序于 {} 发生异常中止，并在 {} 自动重启 失败'.format(
                            app_name, self.event_dict[app_name], now_str)
                        self.event_dict.pop(app_name)
                    self.send_redis({'content': msg, 'monitor_list': ['disk', 'memory']})

                childutils.listener.ok(self.stdout)
                continue

            if int(pheaders['expected']):
                childutils.listener.ok(self.stdout)
                continue

            self.stderr.write('unexpected exit, mailing\n')
            self.stderr.flush()

            self.event_dict[app_name] = self._get_now()[1]
            msg = '[{}] 程序于 {} 发生异常中止'.format(
                app_name, self.event_dict[app_name], self._get_now()[1])
            self.send_redis({'content': msg, 'monitor_list': ['disk', 'memory']})

            childutils.listener.ok(self.stdout)


def main():
    programs = []
    prog = CrashExceptionWarning(programs, True)
    prog.runforever()


if __name__ == '__main__':
    main()