youtube-dl/lib/framework/init_main.py

import logging
import logging.handlers
import os
import platform
import shutil
import sys
import time
import traceback
from datetime import datetime

from flask import Flask
from flask_cors import CORS
from flask_login import LoginManager, login_required
from flask_socketio import SocketIO
from flask_sqlalchemy import SQLAlchemy
from flaskext.markdown import Markdown
from pytz import timezone, utc

from .init_declare import CustomFormatter, check_api


class Framework:
    __instance = None

    @classmethod
    def get_instance(cls):
        if cls.__instance == None:
            cls.__instance = Framework()
        return cls.__instance


    def __init__(self):
        self.logger = None
        self.app = None
        self.celery = None
        self.db = None
        self.scheduler = None
        self.socketio = None
        self.path_app_root = None
        self.path_data = None
        self.users = {}

        self.__level_unset_logger_list = []
        self.__logger_list = []
        self.__exit_code = -1
        self.login_manager = None
        #self.plugin_instance_list = {}
        #self.plugin_menus = {}
        
        # 그냥 F. 로 접근 하게....
        self.SystemModelSetting = None
        self.Job = None
        self.login_required = login_required
        self.check_api = check_api
        self.__initialize()
     

    def __initialize(self):
        os.environ['FF'] = "true"
        self.__config_initialize("first")
        self.__make_default_dir()
        
        self.logger = self.get_logger(__package__)
        import support

        self.__prepare_starting()
        self.app = Flask(__name__)
        self.__config_initialize('flask')

        self.__init_db()
        
        
        if True or self.config['run_flask']:
            from .scheduler import Job, Scheduler
            self.scheduler = Scheduler(self)
            self.Job = Job

        if self.config['use_gevent']:
            self.socketio = SocketIO(self.app, cors_allowed_origins="*")
        else:
            self.socketio = SocketIO(self.app, cors_allowed_origins="*", async_mode='threading')
        
        CORS(self.app)
        Markdown(self.app)

        self.login_manager = LoginManager()
        self.login_manager.init_app(self.app)
        self.login_manager.login_view = "/system/login"
        
        self.celery = self.__init_celery()
        
        from flask_dropzone import Dropzone
        self.app.config.update(
            DROPZONE_MAX_FILE_SIZE = 102400,
            DROPZONE_TIMEOUT = 5*60*1000,
            #DROPZONE_ALLOWED_FILE_CUSTOM = True,
            #DROPZONE_ALLOWED_FILE_TYPE = 'default, image, audio, video, text, app, *.*',
        )
        self.dropzone = Dropzone(self.app)


    def __init_db(self):
        # https://flask-sqlalchemy.palletsprojects.com/en/3.0.x/config/#flask_sqlalchemy.config.SQLALCHEMY_BINDS
        # 어떤 편법도 불가. db를 사용하지 않아도 파일이 생김.
        db_path = os.path.join(self.config['path_data'], 'db', 'system.db')
        self.app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{db_path}' # 3.0에서 필수
        self.app.config['SQLALCHEMY_BINDS'] = {'system':f'sqlite:///{db_path}'}
        self.app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

        _ = os.path.join(self.config['path_data'], 'plugins')
        plugins = []
        if os.path.exists(_):
            plugins = os.listdir(_)
        
        if self.config['path_dev'] != None and os.path.exists(self.config['path_dev']):
            plugins += os.listdir(self.config['path_dev'])
        
        for package_name in plugins:
            db_path = os.path.join(self.config['path_data'], 'db', f'{package_name}.db')
            self.app.config['SQLALCHEMY_BINDS'][package_name] = f'sqlite:///{db_path}'
        self.db = SQLAlchemy(self.app, session_options={"autoflush": False})  

        
    def __init_celery(self):
        try:
            from celery import Celery

            #if frame.config['use_celery'] == False or platform.system() == 'Windows':
            if self.config['use_celery'] == False:
                raise Exception('no celery')
            
            redis_port = os.environ.get('REDIS_PORT', None)
            if redis_port == None:
                redis_port = self.config.get('redis_port', None)
            if redis_port == None:
                redis_port = '6379'
  

            self.app.config['CELERY_BROKER_URL'] = 'redis://localhost:%s/0' % redis_port
            self.app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:%s/0' % redis_port
            celery = Celery(self.app.name, broker=self.app.config['CELERY_BROKER_URL'], backend=self.app.config['CELERY_RESULT_BACKEND'])
            celery.conf['CELERY_ENABLE_UTC'] = False
            celery.conf.update(
                task_serializer='pickle',
                result_serializer='pickle',
                accept_content=['pickle'],
                timezone='Asia/Seoul'
            )
            from celery import bootsteps
            from click import Option
            celery.user_options['worker'].add(Option(('--config_filepath',), help=''),)
            celery.user_options['worker'].add(Option(('--running_type',), help=''),)
            class CustomArgs(bootsteps.Step):
                def __init__(self, worker, config_filepath=None, running_type=None,  **options):
                    from . import F
                    F.logger.info(f"celery config_filepath: {config_filepath}")
                    F.logger.info(f"celery running_type: {running_type}")
                    #F.logger.info(f"celery running_type: {options}")
            celery.steps['worker'].add(CustomArgs)
        except Exception as e:
            self.logger.error('CELERY!!!')
            self.logger.error(f'Exception:{str(e)}')
            self.logger.error(traceback.format_exc())
            def dummy_func():
                pass
                    
            class celery(object):
                class task(object):
                    def __init__(self, *args, **kwargs):
                        if len(args) > 0:
                            self.f = args[0]
                
                    def __call__(self, *args, **kwargs):
                        if len(args) > 0 and type(args[0]) == type(dummy_func):
                            return args[0]
                        self.f(*args, **kwargs)
        return celery


    def initialize_system(self):
        from system.setup import P
        SystemInstance = P
        try:
            with self.app.app_context():
                self.db.create_all()
        except Exception as e:
            self.logger.error('CRITICAL db.create_all()!!!')
            self.logger.error(f'Exception:{str(e)}')
            self.logger.error(traceback.format_exc())
        self.SystemModelSetting = SystemInstance.ModelSetting 
        SystemInstance.plugin_load()
        self.app.register_blueprint(SystemInstance.blueprint)
        self.config['flag_system_loading'] = True
        self.__config_initialize('member')
        self.__config_initialize('system_loading_after')
       

    def initialize_plugin(self): 
        from system.setup import P as SP

        from .init_web import jinja_initialize
        jinja_initialize(self.app)

        #system.LogicPlugin.custom_plugin_update()
        from .init_plugin import PluginManager
        self.PluginManager = PluginManager
        PluginManager.plugin_init()
        PluginManager.plugin_menus['system'] = {'menu':SP.menu, 'match':False} 
        
        #from .init_menu import init_menu, get_menu_map
        from .init_menu import MenuManager
        MenuManager.init_menu()
        #init_menu(self.plugin_menu)
        #system.SystemLogic.apply_menu_link()

        if self.config['run_flask']:
            if self.config.get('port') == None:
                self.config['port'] = self.SystemModelSetting.get_int('port')

        from . import init_route, log_viewer

        self.__make_default_logger()
        self.logger.info('### LAST')
        self.logger.info(f"### PORT: {self.config.get('port')}")
        self.logger.info('### Now you can access App by webbrowser!!')

    def __prepare_starting(self):
        # 여기서 monkey.patch시 너무 늦다고 문제 발생
        pass
        
        
    ###################################################
    # 환경
    ###################################################
    def __config_initialize(self, mode):
        if mode == "first":
            self.config = {}
            self.config['os'] = platform.system()
            self.config['flag_system_loading'] = False
            #self.config['run_flask'] = True if sys.argv[0].endswith('main.py') else False
            self.config['run_celery'] = True if sys.argv[0].find('celery') != -1 else False
            self.config['run_flask'] = not self.config['run_celery']
            self.config['path_app'] = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
            if self.config['os'] == 'Windows' and self.config['path_app'][0] != '/':
                self.config['path_app'] = self.config['path_app'][0].upper() + self.config['path_app'][1:]
            self.path_app_root = self.config['path_app']
            self.config['path_working'] = os.getcwd()
            if os.environ.get('RUNNING_TYPE') == 'docker':
                self.config['running_type'] = 'docker'
            self.config['export_filepath'] = os.path.join(self.config['path_app'], 'export.sh')
            self.config['exist_export'] = os.path.exists(self.config['export_filepath'])
            self.__process_args()
            self.__load_config()
            self.__init_define()
            self.config['menu_yaml_filepath'] = os.path.join(self.config['path_data'], 'db', 'menu.yaml')
            self.config['notify_yaml_filepath'] = os.path.join(self.config['path_data'], 'db', 'notify.yaml')
        elif mode == "flask":
            self.app.secret_key = os.urandom(24)
            self.app.config['TEMPLATES_AUTO_RELOAD'] = True
            self.app.config['JSON_AS_ASCII'] = False
        elif mode == 'system_loading_after':
            if 'running_type' not in self.config:
                self.config['running_type'] = 'native'


    def __init_define(self):
        self.config['DEFINE'] = {}
        # 이건 필요 없음
        self.config['DEFINE']['MAIN_SERVER_URL'] = 'https://server.sjva.me'


    def __process_args(self):
        # celery 에서 args 처리시 문제 발생.
        if self.config['run_flask']:
            import argparse
            parser = argparse.ArgumentParser()
            parser.add_argument('--config', default='.', help='config filepath. Default: {current folder}/config.yaml')
            parser.add_argument('--repeat', default=0, type=int, help=u'Do not set. This value is set by automatic')
            args = parser.parse_args()
            self.config['arg_repeat'] = args.repeat
            self.config['arg_config'] = args.config
        else:
            # 아주 안좋은 구조..
            # celery user_options으로 configfilepath를 받은 후 처리해야하나, 로그파일 경로 등에서 데이터 폴더 위치를 미리 사용하는 경우가 많다.
            # sys.argv에서 데이터 경로를 바로 가져와서 사용.
            self.config['arg_repeat'] = 0
            #self.config['arg_config'] = sys.argv[-1].split('=')[-1]
            #self.config['arg_config'] = sys.argv[-1].split('=')[-1]
            for tmp in sys.argv:
                if tmp.startswith('--config_filepath'):
                    self.config['arg_config'] = tmp.split('=')[1]
                    #break
                elif tmp.startswith('--running_type'):
                    self.config['running_type'] = tmp.split('=')[1]

            #self.config['arg_config'] = 

    def __load_config(self):
        from .init_declare import read_yaml

        #if self.config['run_flask']:
        if self.config['arg_config'] == '.':
            #self.config['config_filepath'] = os.path.join(self.path_app_root, 'config.yaml')
            self.config['config_filepath'] = os.path.join(self.config['path_working'], 'config.yaml')
        else:
            self.config['config_filepath'] = self.config['arg_config']
        if not os.path.exists(self.config['config_filepath']):
            # celery는 환경변수 사용불가로 native 판단
            # 도커는 celery가 먼저 진입
            # 추후에 변경할 것!!!!!!!!!!!!!!!!! TODO
            #if self.config.get('running_type') == 'docker':
            if self.config.get('running_type') == 'docker':# or os.path.exists('/data'):
                shutil.copy(
                    os.path.join(self.path_app_root, 'files', 'config.yaml.docker'),
                    self.config['config_filepath']
                )
            else:                
                shutil.copy(
                    os.path.join(self.path_app_root, 'files', 'config.yaml.template'),
                    self.config['config_filepath']
                )
        data = read_yaml(self.config['config_filepath'])
        for key, value in data.items():
            if key == 'running_type' and value not in ['termux', 'entware']:
                continue
            self.config[key] = value

        if self.config['path_data'] == '.':
            self.config['path_data'] = self.config['path_working']
            # 예외적으로 현재폴더가 app일 경우 지저분해지는 것을 방지하기 위해 data 로 지정
            if self.config['path_data'] == self.config['path_working']:
                self.config['path_data'] = os.path.join(self.config['path_working'], 'data')
        self.path_data = self.config['path_data'] 

        if self.config.get('use_gevent') == None:
            self.config['use_gevent'] = True
        if self.config.get('use_celery') == None:
            self.config['use_celery'] = True
        if self.config.get('debug') == None:
            self.config['debug'] = False
        if self.config.get('plugin_update') == None:
            self.config['plugin_update'] = True
        if self.config.get('plugin_loading_only_devpath') == None:
            self.config['plugin_loading_only_devpath'] = False
        if self.config.get('plugin_loading_list') == None:
            self.config['plugin_loading_list'] = []
        if self.config.get('plugin_except_list') == None:
            self.config['plugin_except_list'] = []
        if self.config.get('path_dev') == None:
            self.config['path_dev'] = None
        

    def __make_default_dir(self):
        os.makedirs(self.config['path_data'], exist_ok=True)
        tmp = os.path.join(self.config['path_data'], 'tmp')
        try:
            import shutil
            if os.path.exists(tmp):
                shutil.rmtree(tmp)
        except:
            pass
        sub = ['db', 'log', 'tmp']
        for item in sub:
            tmp = os.path.join(self.config['path_data'], item)
            os.makedirs(tmp, exist_ok=True)
        
    ###################################################



    
    ###################################################
    # 로그
    ###################################################
    def get_logger(self, name):
        logger = logging.getLogger(name)
        if not logger.handlers:
            level = logging.DEBUG
            try:
                if self.config['flag_system_loading']:
                    try:
                        from system import SystemModelSetting
                        level = SystemModelSetting.get_int('log_level')
                    except:
                        level = logging.DEBUG
                    if self.__level_unset_logger_list is not None:
                        for item in self.__level_unset_logger_list:
                            item.setLevel(level)
                        self.__level_unset_logger_list = None
                else:
                    self.__level_unset_logger_list.append(logger)
                if name.startswith('apscheduler'):
                    level = logging.CRITICAL
                else:
                    self.__logger_list.append(logger)
            except:
                pass
            logger.setLevel(level)
            file_formatter = logging.Formatter(u'[%(asctime)s|%(levelname)s|%(filename)s:%(lineno)s] %(message)s')
            def customTime(*args):
                utc_dt = utc.localize(datetime.utcnow())
                my_tz = timezone("Asia/Seoul")
                converted = utc_dt.astimezone(my_tz) 
                return converted.timetuple()

            file_formatter.converter = customTime
            file_max_bytes = 1 * 1024 * 1024 
            fileHandler = logging.handlers.RotatingFileHandler(filename=os.path.join(self.path_data, 'log', f'{name}.log'), maxBytes=file_max_bytes, backupCount=5, encoding='utf8', delay=True)
            streamHandler = logging.StreamHandler() 

            # handler에 fommater 세팅 
            fileHandler.setFormatter(file_formatter) 
            streamHandler.setFormatter(CustomFormatter()) 
            
            # Handler를 logging에 추가
            logger.addHandler(fileHandler)
            logger.addHandler(streamHandler)
        return logger 


    def __make_default_logger(self):
        self.get_logger('apscheduler.scheduler')
        self.get_logger('apscheduler.executors.default')
        try: logging.getLogger('socketio').setLevel(logging.ERROR)
        except: pass
        try: logging.getLogger('engineio').setLevel(logging.ERROR)
        except: pass
        try: logging.getLogger('apscheduler.scheduler').setLevel(logging.ERROR)
        except: pass
        try: logging.getLogger('apscheduler.executors.default').setLevel(logging.ERROR)
        except: pass
        try: logging.getLogger('werkzeug').setLevel(logging.ERROR)
        except: pass

    def set_level(self, level):
        try:
            for l in self.__logger_list:
                l.setLevel(level)
            self.__make_default_logger()
        except:
            pass    
    ###################################################
    

    def start(self):
        host = '0.0.0.0'
        for i in range(10): 
            try: 
                #self.logger.debug(d(self.config))
                self.socketio.run(self.app, host=host, port=self.config['port'], debug=self.config['debug'], use_reloader=self.config['debug'])
                self.logger.warning(f"EXIT CODE : {self.__exit_code}")
                # 2021-05-18  
                if self.config['running_type'] in ['termux', 'entware']:
                    os._exit(self.__exit_code)
                else:
                    if self.__exit_code != -1:
                        sys.exit(self.__exit_code)
                    else:
                        self.logger.warning(f"framework.exit_code is -1")
                break
            except Exception as exception:
                self.logger.error(f"Start ERROR : {str(exception)}")
                host = '127.0.0.1'  
                time.sleep(10*i)
                continue  
            except KeyboardInterrupt: 
                self.logger.error('KeyboardInterrupt !!')
            #except SystemExit: 
            #    return 
                #sys.exit(self.__exit_code)


    # system 플러그인에서 콜
    def restart(self):
        self.__exit_code = 1
        self.__app_close()

    def shutdown(self):
        self.__exit_code = 0
        self.__app_close()

    def __app_close(self):
        try:
            from support import SupportSubprocess
            SupportSubprocess.all_process_close()
            from .init_plugin import PluginManager
            PluginManager.plugin_unload()
            self.socketio.stop()
        except Exception as exception: 
            self.logger.error('Exception:%s', exception)
            self.logger.error(traceback.format_exc())

    def get_recent_version(self):
        try:
            import requests
            url = f"{self.config['DEFINE']['MAIN_SERVER_URL']}/version"
            self.config['recent_version'] = requests.get(url).text
            return True
        except Exception as e:
            self.logger.error(f'Exception:{str(e)}')
            self.logger.error(traceback.format_exc())
            self.config['recent_version'] =  '확인 실패'
        return False