From e7bb53c33b7acb4affa9f4377803a9a596f48400 Mon Sep 17 00:00:00 2001 From: projectdx Date: Sat, 12 Nov 2022 23:47:21 +0900 Subject: [PATCH] 2022.11.12 (01. bug fixed) --- lib/ffmpeg_queue_v1.py | 370 ++++++ mod_anilife.py | 29 +- mod_linkkf.py | 704 +++++++++- mod_ohli24.py | 241 +++- templates/anime_downloader_anilife_queue.html | 265 ++-- .../anime_downloader_anilife_search.html | 67 +- templates/anime_downloader_linkkf_queue.html | 266 ++-- .../anime_downloader_linkkf_request.html | 55 + templates/anime_downloader_linkkf_search.html | 134 +- templates/anime_downloader_ohli24_queue.html | 266 ++-- .../anime_downloader_ohli24_request.html | 1178 +++++++++-------- templates/anime_downloader_ohli24_search.html | 21 +- 12 files changed, 2522 insertions(+), 1074 deletions(-) create mode 100644 lib/ffmpeg_queue_v1.py diff --git a/lib/ffmpeg_queue_v1.py b/lib/ffmpeg_queue_v1.py new file mode 100644 index 0000000..6a571aa --- /dev/null +++ b/lib/ffmpeg_queue_v1.py @@ -0,0 +1,370 @@ +import abc +import os +import queue +import threading +import time +import traceback +from datetime import datetime + +import requests +# from flaskfarm.lib.plugin import get_model_setting +from flaskfarm.lib.support.expand.ffmpeg import SupportFfmpeg +# from flaskfarm.lib.system.setup import SystemModelSetting +from flaskfarm.lib.tool import ToolUtil +# from flaskfarm.lib.system.setup import P as SM +# from flaskfarm.lib.system.mod_setting import ModuleSetting as SM + +from ..setup import * + + +class FfmpegQueueEntity(abc.ABCMeta('ABC', (object,), {'__slots__': ()})): + + def __init__(self, P, module_logic, info): + self.P = P + # SupportFfmpeg.initialize() + self.module_logic = module_logic + self.entity_id = -1 # FfmpegQueueEntity.static_index + self.info = info + self.url = None + self.ffmpeg_status = -1 + self.ffmpeg_status_kor = u'대기중' + self.ffmpeg_percent = 0 + self.ffmpeg_arg = None + self.cancel = False + self.created_time = datetime.now().strftime('%m-%d %H:%M:%S') + self.savepath = None + self.filename = None + self.filepath = None + self.quality = None + self.headers = None + # FfmpegQueueEntity.static_index += 1 + # FfmpegQueueEntity.entity_list.append(self) + + def get_video_url(self): + return self.url + + def get_video_filepath(self): + return self.filepath + + @abc.abstractmethod + def refresh_status(self): + pass + + @abc.abstractmethod + def info_dict(self, tmp): + pass + + def download_completed(self): + pass + + def as_dict(self): + tmp = {} + tmp['entity_id'] = self.entity_id + tmp['url'] = self.url + tmp['ffmpeg_status'] = self.ffmpeg_status + tmp['ffmpeg_status_kor'] = self.ffmpeg_status_kor + tmp['ffmpeg_percent'] = self.ffmpeg_percent + tmp['ffmpeg_arg'] = self.ffmpeg_arg + tmp['cancel'] = self.cancel + tmp['created_time'] = self.created_time # .strftime('%m-%d %H:%M:%S') + tmp['savepath'] = self.savepath + tmp['filename'] = self.filename + tmp['filepath'] = self.filepath + tmp['quality'] = self.quality + # tmp['current_speed'] = self.ffmpeg_arg['current_speed'] if self.ffmpeg_arg is not None else '' + tmp = self.info_dict(tmp) + return tmp + + +class FfmpegQueue(object): + + def __init__(self, P, max_ffmpeg_count): + + self.P = P + self.static_index = 1 + self.entity_list = [] + self.current_ffmpeg_count = 0 + self.download_queue = None + self.download_thread = None + self.max_ffmpeg_count = max_ffmpeg_count + if self.max_ffmpeg_count is None or self.max_ffmpeg_count == '': + self.max_ffmpeg_count = 1 + + def queue_start(self): + try: + if self.download_queue is None: + self.download_queue = queue.Queue() + if self.download_thread is None: + self.download_thread = threading.Thread(target=self.download_thread_function, args=()) + self.download_thread.daemon = True + # todo: + # self.download_thread.start() + except Exception as exception: + self.P.logger.error(f'Exception: {exception}') + self.P.logger.error(traceback.format_exc()) + + def download_thread_function(self): + while True: + try: + while True: + try: + if self.current_ffmpeg_count < self.max_ffmpeg_count: + break + time.sleep(5) + except Exception as exception: + self.P.logger.error(f'Exception: {exception}') + self.P.logger.error(traceback.format_exc()) + self.P.logger.error('current_ffmpeg_count : %s', self.current_ffmpeg_count) + self.P.logger.error('max_ffmpeg_count : %s', self.max_ffmpeg_count) + break + entity = self.download_queue.get() + if entity.cancel: + continue + + # from .logic_ani24 import LogicAni24 + # entity.url = LogicAni24.get_video_url(entity.info['code']) + video_url = entity.get_video_url() + if video_url is None: + entity.ffmpeg_status_kor = 'URL실패' + entity.refresh_status() + # plugin.socketio_list_refresh() + continue + + # import ffmpeg + + # max_pf_count = 0 + # save_path = ModelSetting.get('download_path') + # if ModelSetting.get('auto_make_folder') == 'True': + # program_path = os.path.join(save_path, entity.info['filename'].split('.')[0]) + # save_path = program_path + # try: + # if not os.path.exists(save_path): + # os.makedirs(save_path) + # except: + # logger.debug('program path make fail!!') + # 파일 존재여부 체크 + P.logger.info(entity.info) + filepath = entity.get_video_filepath() + P.logger.debug(f'filepath:: {filepath}') + if os.path.exists(filepath): + entity.ffmpeg_status_kor = '파일 있음' + entity.ffmpeg_percent = 100 + entity.refresh_status() + # plugin.socketio_list_refresh() + continue + dirname = os.path.dirname(filepath) + # filename = os.path.f + if not os.path.exists(dirname): + os.makedirs(dirname) + # f = ffmpeg.Ffmpeg(video_url, os.path.basename(filepath), plugin_id=entity.entity_id, listener=self.ffmpeg_listener, call_plugin=self.P.package_name, save_path=dirname, headers=entity.headers) + # print(filepath) + # print(os.path.basename(filepath)) + # print(dirname) + # aa_sm = get_model_setting("system", P.logger) + P.logger.debug(P) + # P.logger.debug(P.system_setting.get("port")) + + ffmpeg = SupportFfmpeg(video_url, str(os.path.basename(filepath)), + callback_function=self.callback_function, + max_pf_count=0, save_path=ToolUtil.make_path(dirname), timeout_minute=60, + ) + # + # todo: 임시로 start() 중지 + # ffmpeg.start() + self.current_ffmpeg_count += 1 + self.download_queue.task_done() + + except Exception as exception: + self.P.logger.error('Exception:%s', exception) + self.P.logger.error(traceback.format_exc()) + + def callback_function(self, **args): + refresh_type = None + if args['type'] == 'status_change': + if args['status'] == SupportFfmpeg.Status.DOWNLOADING: + refresh_type = 'status_change' + elif args['status'] == SupportFfmpeg.Status.COMPLETED: + refresh_type = 'status_change' + elif args['status'] == SupportFfmpeg.Status.READY: + data = {'type': 'info', + 'msg': '다운로드중 Duration(%s)' % args['data']['duration_str'] + '
' + args['data'][ + 'save_fullpath'], 'url': '/ffmpeg/download/list'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'add' + elif args['type'] == 'last': + if args['status'] == SupportFfmpeg.Status.WRONG_URL: + data = {'type': 'warning', 'msg': '잘못된 URL입니다'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'add' + elif args['status'] == SupportFfmpeg.Status.WRONG_DIRECTORY: + data = {'type': 'warning', 'msg': '잘못된 디렉토리입니다.
' + args['data']['save_fullpath']} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'add' + elif args['status'] == SupportFfmpeg.Status.ERROR or args['status'] == SupportFfmpeg.Status.EXCEPTION: + data = {'type': 'warning', 'msg': '다운로드 시작 실패.
' + args['data']['save_fullpath']} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'add' + elif args['status'] == SupportFfmpeg.Status.USER_STOP: + data = {'type': 'warning', 'msg': '다운로드가 중지 되었습니다.
' + args['data']['save_fullpath'], + 'url': '/ffmpeg/download/list'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'last' + elif args['status'] == SupportFfmpeg.Status.COMPLETED: + data = {'type': 'success', 'msg': '다운로드가 완료 되었습니다.
' + args['data']['save_fullpath'], + 'url': '/ffmpeg/download/list'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'last' + elif args['status'] == SupportFfmpeg.Status.TIME_OVER: + data = {'type': 'warning', 'msg': '시간초과로 중단 되었습니다.
' + args['data']['save_fullpath'], + 'url': '/ffmpeg/download/list'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'last' + elif args['status'] == SupportFfmpeg.Status.PF_STOP: + data = {'type': 'warning', 'msg': 'PF초과로 중단 되었습니다.
' + args['data']['save_fullpath'], + 'url': '/ffmpeg/download/list'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'last' + elif args['status'] == SupportFfmpeg.Status.FORCE_STOP: + data = {'type': 'warning', 'msg': '강제 중단 되었습니다.
' + args['data']['save_fullpath'], + 'url': '/ffmpeg/download/list'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'last' + elif args['status'] == SupportFfmpeg.Status.HTTP_FORBIDDEN: + data = {'type': 'warning', 'msg': '403에러로 중단 되었습니다.
' + args['data']['save_fullpath'], + 'url': '/ffmpeg/download/list'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'last' + elif args['status'] == SupportFfmpeg.Status.ALREADY_DOWNLOADING: + data = {'type': 'warning', 'msg': '임시파일폴더에 파일이 있습니다.
' + args['data']['temp_fullpath'], + 'url': '/ffmpeg/download/list'} + socketio.emit("notify", data, namespace='/framework', broadcast=True) + refresh_type = 'last' + elif args['type'] == 'normal': + if args['status'] == SupportFfmpeg.Status.DOWNLOADING: + refresh_type = 'status' + # P.logger.info(refresh_type) + self.socketio_callback(refresh_type, args['data']) + + def ffmpeg_listener(self, **arg): + import ffmpeg + entity = self.get_entity_by_entity_id(arg['plugin_id']) + if entity is None: + return + if arg['type'] == 'status_change': + if arg['status'] == ffmpeg.Status.DOWNLOADING: + pass + elif arg['status'] == ffmpeg.Status.COMPLETED: + entity.donwload_completed() + elif arg['status'] == ffmpeg.Status.READY: + pass + elif arg['type'] == 'last': + self.current_ffmpeg_count += -1 + elif arg['type'] == 'log': + pass + elif arg['type'] == 'normal': + pass + + entity.ffmpeg_arg = arg + entity.ffmpeg_status = int(arg['status']) + entity.ffmpeg_status_kor = str(arg['status']) + entity.ffmpeg_percent = arg['data']['percent'] + entity.ffmpeg_arg['status'] = str(arg['status']) + # self.P.logger.debug(arg) + # import plugin + # arg['status'] = str(arg['status']) + # plugin.socketio_callback('status', arg) + entity.refresh_status() + + # FfmpegQueueEntity.static_index += 1 + # FfmpegQueueEntity.entity_list.append(self) + + def add_queue(self, entity): + try: + # entity = QueueEntity.create(info) + # if entity is not None: + # LogicQueue.download_queue.put(entity) + # return True + entity.entity_id = self.static_index + self.static_index += 1 + self.entity_list.append(entity) + self.download_queue.put(entity) + return True + except Exception as exception: + self.P.logger.error('Exception:%s', exception) + self.P.logger.error(traceback.format_exc()) + return False + + def set_max_ffmpeg_count(self, max_ffmpeg_count): + self.max_ffmpeg_count = max_ffmpeg_count + + def get_max_ffmpeg_count(self): + return self.max_ffmpeg_count + + def command(self, cmd, entity_id): + self.P.logger.debug('command :%s %s', cmd, entity_id) + ret = {} + try: + if cmd == 'cancel': + self.P.logger.debug('command :%s %s', cmd, entity_id) + entity = self.get_entity_by_entity_id(entity_id) + if entity is not None: + if entity.ffmpeg_status == -1: + entity.cancel = True + entity.ffmpeg_status_kor = "취소" + # entity.refresh_status() + ret['ret'] = 'refresh' + elif entity.ffmpeg_status != 5: + ret['ret'] = 'notify' + ret['log'] = '다운로드중 상태가 아닙니다.' + else: + idx = entity.ffmpeg_arg['data']['idx'] + import ffmpeg + ffmpeg.Ffmpeg.stop_by_idx(idx) + entity.refresh_status() + ret['ret'] = 'refresh' + elif cmd == 'reset': + if self.download_queue is not None: + with self.download_queue.mutex: + self.download_queue.queue.clear() + for _ in self.entity_list: + if _.ffmpeg_status == 5: + import ffmpeg + idx = _.ffmpeg_arg['data']['idx'] + ffmpeg.Ffmpeg.stop_by_idx(idx) + self.entity_list = [] + ret['ret'] = 'refresh' + elif cmd == 'delete_completed': + new_list = [] + for _ in self.entity_list: + if _.ffmpeg_status_kor in [u'파일 있음', u'취소', u'사용자중지']: + continue + if _.ffmpeg_status != 7: + new_list.append(_) + self.entity_list = new_list + ret['ret'] = 'refresh' + elif cmd == 'remove': + new_list = [] + for _ in self.entity_list: + if _.entity_id == entity_id: + continue + new_list.append(_) + self.entity_list = new_list + ret['ret'] = 'refresh' + return ret + except Exception as exception: + self.P.logger.error('Exception:%s', exception) + self.P.logger.error(traceback.format_exc()) + + def get_entity_by_entity_id(self, entity_id): + for _ in self.entity_list: + if _.entity_id == entity_id: + return _ + return None + + def get_entity_list(self): + ret = [] + P.logger.debug(self) + for x in self.entity_list: + tmp = x.as_dict() + ret.append(tmp) + return ret diff --git a/mod_anilife.py b/mod_anilife.py index e4d9ac1..3c5ca66 100644 --- a/mod_anilife.py +++ b/mod_anilife.py @@ -49,7 +49,7 @@ from framework import F from plugin import ( PluginModuleBase ) -from .lib._ffmpeg_queue import FfmpegQueueEntity, FfmpegQueue +from .lib.ffmpeg_queue import FfmpegQueueEntity, FfmpegQueue from .lib.crawler import Crawler # from tool_base import d @@ -459,12 +459,19 @@ class LogicAniLife(PluginModuleBase): data = [] cate = request.form["type"] page = request.form["page"] + try: + data = self.get_anime_info(cate, page) + logger.debug(data) + if data is not None: + return jsonify( + {"ret": "success", "cate": cate, "page": page, "data": data} + ) + else: + return jsonify({"ret": "error", "data": data}) - data = self.get_anime_info(cate, page) - # self.current_data = data - return jsonify( - {"ret": "success", "cate": cate, "page": page, "data": data} - ) + except Exception as e: + print("error catch") + return jsonify({"ret": "error", "data": data}) elif sub == "complete_list": data = [] @@ -828,7 +835,11 @@ class LogicAniLife(PluginModuleBase): } payload = json.dumps(post_data) logger.debug(payload) - response_data = requests.post(url="http://localhost:7070/get_html_playwright", data=payload) + try: + response_data = requests.post(url="http://localhost:7070/get_html_playwright", data=payload) + except Exception as e: + logger.error(f"Exception: {str(e)}") + return LogicAniLife.episode_url = response_data.json()["url"] logger.info(response_data.json()["url"]) @@ -857,7 +868,7 @@ class LogicAniLife(PluginModuleBase): for item in tmp_items: entity = {} entity["link"] = item.xpath(".//a/@href")[0] - logger.debug(entity["link"]) + # logger.debug(entity["link"]) p = re.compile(r"^[http?s://]+[a-zA-Z0-9-]+/[a-zA-Z0-9-_.?=]+$") # print(p.match(entity["link"]) != None) @@ -865,8 +876,6 @@ class LogicAniLife(PluginModuleBase): entity["link"] = P.ModelSetting.get("anilife_url") + entity["link"] # real_url = LogicAniLife.get_real_link(url=entity["link"]) - # logger.debug(entity["link"]) - entity["code"] = entity["link"].split("/")[-1] entity["title"] = item.xpath(".//div[@class='tt']/text()")[0].strip() entity["image_link"] = item.xpath(".//div[@class='limit']/img/@src")[ diff --git a/mod_linkkf.py b/mod_linkkf.py index 9a39e5e..6d737c6 100644 --- a/mod_linkkf.py +++ b/mod_linkkf.py @@ -11,6 +11,10 @@ import re import sys import traceback from datetime import datetime +import random +import time +import urllib +from urllib.parse import urlparse import PIL.Image # third-party @@ -18,12 +22,34 @@ import requests from bs4 import BeautifulSoup # third-party from flask import jsonify, render_template, request +from flaskfarm.lib.support.expand.ffmpeg import SupportFfmpeg # sjva 공용 from framework import db, path_data, scheduler from lxml import html from plugin import PluginModuleBase from requests_cache import CachedSession +packages = ["beautifulsoup4", "requests-cache", "cloudscraper"] + +for package in packages: + try: + import package + + except ModuleNotFoundError: + if package == "playwright": + pass + # os.system(f"pip3 install playwright") + # os.system(f"playwright install") + except ImportError: + # main(["install", package]) + if package == "playwright": + pass + # os.system(f"pip3 install {package}") + # os.system(f"playwright install") + else: + os.system(f"pip3 install {package}") + +from anime_downloader.lib.ffmpeg_queue_v1 import FfmpegQueueEntity, FfmpegQueue from anime_downloader.lib.util import Util # 패키지 # from .plugin import P @@ -37,31 +63,17 @@ from anime_downloader.setup import * logger = P.logger +name = 'linkkf' class LogicLinkkf(PluginModuleBase): - db_default = { - "linkkf_db_version": "1", - "linkkf_url": "https://linkkf.app", - "linkkf_download_path": os.path.join(path_data, P.package_name, "linkkf"), - "linkkf_auto_make_folder": "True", - "linkkf_auto_make_season_folder": "True", - "linkkf_finished_insert": "[완결]", - "linkkf_max_ffmpeg_process_count": "1", - "linkkf_order_desc": "False", - "linkkf_auto_start": "False", - "linkkf_interval": "* 5 * * *", - "linkkf_auto_mode_all": "False", - "linkkf_auto_code_list": "all", - "linkkf_current_code": "", - "linkkf_uncompleted_auto_enqueue": "False", - "linkkf_image_url_prefix_series": "", - "linkkf_image_url_prefix_episode": "", - "linkkf_discord_notify": "True", - } current_headers = None current_data = None referer = None + download_queue = None + download_thread = None + current_download_count = 0 + cache_path = os.path.dirname(__file__) session = requests.Session() @@ -79,9 +91,34 @@ class LogicLinkkf(PluginModuleBase): def __init__(self, P): super(LogicLinkkf, self).__init__(P, "setting", scheduler_desc="linkkf 자동 다운로드") - self.name = "linkkf" + self.queue = None + self.name = name + self.db_default = { + "linkkf_db_version": "1", + "linkkf_url": "https://linkkf.app", + f"{self.name}_recent_code": "", + "linkkf_download_path": os.path.join(path_data, P.package_name, "linkkf"), + "linkkf_save_path": os.path.join(path_data, P.package_name, "linkkf"), + "linkkf_auto_make_folder": "True", + "linkkf_auto_make_season_folder": "True", + "linkkf_finished_insert": "[완결]", + "linkkf_max_ffmpeg_process_count": "2", + f"{self.name}_max_download_count": "2", + f"{self.name}_quality": "720p", + "linkkf_order_desc": "False", + "linkkf_auto_start": "False", + "linkkf_interval": "* 5 * * *", + "linkkf_auto_mode_all": "False", + "linkkf_auto_code_list": "all", + "linkkf_current_code": "", + "linkkf_uncompleted_auto_enqueue": "False", + "linkkf_image_url_prefix_series": "", + "linkkf_image_url_prefix_episode": "", + "linkkf_discord_notify": "True", + } # default_route_socketio(P, self) default_route_socketio_module(self, attach='/setting') + self.current_data = None def process_menu(self, sub, req): arg = P.ModelSetting.to_dict() @@ -134,7 +171,7 @@ class LogicLinkkf(PluginModuleBase): dummy_data = {"ret": "success", "data": data} return jsonify(data) except Exception as e: - logger.error("Exception:%s", e) + logger.error(f"Exception: {str(e)}") logger.error(traceback.format_exc()) elif sub == "complete_list": pass @@ -163,6 +200,7 @@ class LogicLinkkf(PluginModuleBase): info = json.loads(request.form["data"]) logger.info(f"info:: {info}") ret["ret"] = self.add(info) + return jsonify(ret) elif sub == "entity_list": pass elif sub == "queue_command": @@ -177,9 +215,382 @@ class LogicLinkkf(PluginModuleBase): pass except Exception as e: - P.logger.error("Exception:%s", e) + P.logger.error(f"Exception: {str(e)}") P.logger.error(traceback.format_exc()) + @staticmethod + def get_html(url, cached=False): + + try: + if LogicLinkkf.referer is None: + LogicLinkkf.referer = "https://linkkf.app/" + + # return LogicLinkkfYommi.get_html_requests(url) + return LogicLinkkf.get_html_cloudflare(url) + + except Exception as e: + logger.error("Exception:%s", e) + logger.error(traceback.format_exc()) + + @staticmethod + def get_html_cloudflare(url, cached=False): + logger.debug(f"cloudflare protection bypass {'=' * 30}") + + user_agents_list = [ + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36", + ] + # ua = UserAgent(verify_ssl=False) + + LogicLinkkf.headers["User-Agent"] = random.choice(user_agents_list) + + LogicLinkkf.headers["Referer"] = LogicLinkkf.referer + + # logger.debug(f"headers:: {LogicLinkkfYommi.headers}") + + if LogicLinkkf.session is None: + LogicLinkkf.session = requests.Session() + + # LogicLinkkfYommi.session = requests.Session() + # re_sess = requests.Session() + # logger.debug(LogicLinkkfYommi.session) + + # sess = cloudscraper.create_scraper( + # # browser={"browser": "firefox", "mobile": False}, + # browser={"browser": "chrome", "mobile": False}, + # debug=True, + # sess=LogicLinkkfYommi.session, + # delay=10, + # ) + # scraper = cloudscraper.create_scraper(sess=re_sess) + scraper = cloudscraper.create_scraper( + # debug=True, + delay=10, + sess=LogicLinkkf.session, + browser={ + "custom": "linkkf", + }, + ) + + # print(scraper.get(url, headers=LogicLinkkfYommi.headers).content) + # print(scraper.get(url).content) + # return scraper.get(url, headers=LogicLinkkfYommi.headers).content + # logger.debug(LogicLinkkfYommi.headers) + return scraper.get( + url, + headers=LogicLinkkf.headers, + timeout=10, + ).content.decode("utf8", errors="replace") + + @staticmethod + def get_video_url_from_url(url, url2): + video_url = None + referer_url = None + vtt_url = None + LogicLinkkf.referer = url2 + # logger.info("dx download url : %s , url2 : %s" % (url, url2)) + # logger.debug(LogicLinkkfYommi.referer) + + try: + if "ani1" in url2: + # kfani 계열 처리 => 방문해서 m3u8을 받아온다. + logger.debug("ani1 routine=========================") + LogicLinkkf.referer = "https://linkkf.app" + # logger.debug(f"url2: {url2}") + ani1_html = LogicLinkkf.get_html(url2) + + tree = html.fromstring(ani1_html) + option_url = tree.xpath("//select[@id='server-list']/option[1]/@value") + + # logger.debug(f"option_url:: {option_url}") + + data = LogicLinkkf.get_html(option_url[0]) + # print(type(data)) + regex2 = r'"([^\"]*m3u8)"|]+src=\"([^"]+)' + + temp_url = re.findall(regex2, data)[0] + video_url = "" + ref = "https://ani1.app" + for i in temp_url: + if i is None: + continue + video_url = i + # video_url = '{1} -headers \'Referer: "{0}"\' -user_agent "Mozilla/5.0 (Windows NT 10.0; Win64; + # x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3554.0 Safari/537.36"'.format(ref, + # video_url) + + data_tree = html.fromstring(data) + # print(data_tree.xpath("//video/source/@src")) + vtt_elem = data_tree.xpath("//track/@src")[0] + # vtt_elem = data_tree.xpath("//*[contains(@src, '.vtt']")[0] + + # print(vtt_elem) + + match = re.compile( + r".*?.vtt)\"", re.MULTILINE + ).search(data) + + vtt_url = match.group("vtt_url") + + referer_url = "https://kfani.me/" + + elif "kfani" in url2: + # kfani 계열 처리 => 방문해서 m3u8을 받아온다. + logger.debug("kfani routine=================================") + LogicLinkkf.referer = url2 + # logger.debug(f"url2: {url2}") + data = LogicLinkkf.get_html(url2) + # logger.info("dx: data", data) + regex2 = r'"([^\"]*m3u8)"|]+src=\"([^"]+)' + + temp_url = re.findall(regex2, data)[0] + video_url = "" + ref = "https://kfani.me" + for i in temp_url: + if i is None: + continue + video_url = i + # video_url = '{1} -headers \'Referer: "{0}"\' -user_agent "Mozilla/5.0 (Windows NT 10.0; Win64; + # x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3554.0 Safari/537.36"'.format(ref, + # video_url) + + match = re.compile( + r".*?.vtt)", re.MULTILINE + ).search(data) + + vtt_url = match.group("vtt_url") + logger.info("vtt_url: %s", vtt_url) + + referer_url = url2 + + elif "kftv" in url2: + # kftv 계열 처리 => url의 id로 https://yt.kftv.live/getLinkStreamMd5/df6960891d226e24b117b850b44a2290 페이지 + # 접속해서 json 받아오고, json에서 url을 추출해야함 + if "=" in url2: + md5 = urlparse.urlparse(url2).query.split("=")[1] + elif "embedplay" in url2: + md5 = url2.split("/")[-1] + url3 = "https://yt.kftv.live/getLinkStreamMd5/" + md5 + # logger.info("download url : %s , url3 : %s" % (url, url3)) + data3 = LogicLinkkf.get_html(url3) + data3dict = json.loads(data3) + # print(data3dict) + video_url = data3dict[0]["file"] + + elif "k40chan" in url2: + # k40chan 계열 처리 => 방문해서 m3u8을 받아온다. + # k45734 님 소스 반영 (확인은 안해봄 잘 동작할꺼라고 믿고,) + logger.debug("k40chan routine=================================") + LogicLinkkf.referer = url2 + data = LogicLinkkf.get_html(url2) + + regex2 = r'"([^\"]*m3u8)"|]+src=\"([^"]+)' + + temp_url = re.findall(regex2, data)[0] + video_url = "" + # ref = "https://kfani.me" + for i in temp_url: + if i is None: + continue + video_url = i + + match = re.compile(r".*?.vtt)").search(data) + vtt_url = match.group("vtt_url") + + referer_url = url2 + + elif "linkkf" in url2: + logger.deubg("linkkf routine") + # linkkf 계열 처리 => URL 리스트를 받아오고, 하나 골라 방문 해서 m3u8을 받아온다. + referer_url = url2 + data2 = LogicLinkkf.get_html(url2) + # print(data2) + regex = r"cat1 = [^\[]*([^\]]*)" + cat = re.findall(regex, data2)[0] + # logger.info("cat: %s", cat) + regex = r"\"([^\"]*)\"" + url3s = re.findall(regex, cat) + url3 = random.choice(url3s) + # logger.info("url3: %s", url3) + # logger.info("download url : %s , url3 : %s" % (url, url3)) + if "kftv" in url3: + return LogicLinkkf.get_video_url_from_url(url2, url3) + elif url3.startswith("/"): + url3 = urlparse.urljoin(url2, url3) + print("url3 = ", url3) + LogicLinkkf.referer = url2 + data3 = LogicLinkkf.get_html(url3) + # logger.info('data3: %s', data3) + # regex2 = r'"([^\"]*m3u8)"' + regex2 = r'"([^\"]*mp4|m3u8)"' + video_url = re.findall(regex2, data3)[0] + # logger.info('video_url: %s', video_url) + referer_url = url3 + + else: + logger.error("새로운 유형의 url 발생! %s %s %s" % (url, url2, url3)) + elif "kakao" in url2: + # kakao 계열 처리, 외부 API 이용 + payload = {"inputUrl": url2} + kakao_url = ( + "http://webtool.cusis.net/wp-pages/download-kakaotv-video/video.php" + ) + data2 = requests.post( + kakao_url, + json=payload, + headers={ + "referer": "http://webtool.cusis.net/download-kakaotv-video/" + }, + ).content + time.sleep(3) # 서버 부하 방지를 위해 단시간에 너무 많은 URL전송을 하면 IP를 차단합니다. + url3 = json.loads(data2) + # logger.info("download url2 : %s , url3 : %s" % (url2, url3)) + video_url = url3 + elif "#V" in url2: # V 패턴 추가 + print("#v routine") + + data2 = LogicLinkkf.get_html(url2) + + regex = r"cat1 = [^\[]*([^\]]*)" + cat = re.findall(regex, data2)[0] + regex = r"\"([^\"]*)\"" + url3s = re.findall(regex, cat) + url3 = random.choice(url3s) + # logger.info("download url : %s , url3 : %s" % (url, url3)) + if "kftv" in url3: + return LogicLinkkf.get_video_url_from_url(url2, url3) + elif url3.startswith("/"): + url3 = urlparse.urljoin(url2, url3) + LogicLinkkf.referer = url2 + data3 = LogicLinkkf.get_html(url3) + + regex2 = r'"([^\"]*mp4)"' + video_url = re.findall(regex2, data3)[0] + else: + logger.error("새로운 유형의 url 발생! %s %s %s" % (url, url2, url3)) + + elif "#M2" in url2: + LogicLinkkf.referer = url2 + data2 = LogicLinkkf.get_html(url2) + # print(data2) + + regex = r"cat1 = [^\[]*([^\]]*)" + cat = re.findall(regex, data2)[0] + regex = r"\"([^\"]*)\"" + url3s = re.findall(regex, cat) + url3 = random.choice(url3s) + # logger.info("download url : %s , url3 : %s" % (url, url3)) + if "kftv" in url3: + return LogicLinkkf.get_video_url_from_url(url2, url3) + elif url3.startswith("/"): + url3 = urlparse.urljoin(url2, url3) + LogicLinkkf.referer = url2 + data3 = LogicLinkkf.get_html(url3) + # print("내용: %s", data3) + # logger.info("movie content: %s", data3) + # regex2 = r'"([^\"]*m3u8)"' + regex2 = r'"([^\"]*mp4)"' + video_url = re.findall(regex2, data3)[0] + else: + logger.error("새로운 유형의 url 발생! %s %s %s" % (url, url2, url3)) + elif "😀#i" in url2: + LogicLinkkf.referer = url2 + data2 = LogicLinkkf.get_html(url2) + # logger.info(data2) + + regex = r"cat1 = [^\[]*([^\]]*)" + cat = re.findall(regex, data2)[0] + regex = r"\"([^\"]*)\"" + url3s = re.findall(regex, cat) + url3 = random.choice(url3s) + # logger.info("download url : %s , url3 : %s" % (url, url3)) + + elif "#k" in url2: + data2 = LogicLinkkf.get_html(url2) + # logger.info(data2) + + regex = r"cat1 = [^\[]*([^\]]*)" + cat = re.findall(regex, data2)[0] + regex = r"\"([^\"]*)\"" + url3s = re.findall(regex, cat) + url3 = random.choice(url3s) + # logger.info("download url : %s , url3 : %s" % (url, url3)) + + elif "#k2" in url2: + data2 = LogicLinkkf.get_html(url2) + # logger.info(data2) + + regex = r"cat1 = [^\[]*([^\]]*)" + cat = re.findall(regex, data2)[0] + regex = r"\"([^\"]*)\"" + url3s = re.findall(regex, cat) + url3 = random.choice(url3s) + # logger.info("download url : %s , url3 : %s" % (url, url3)) + elif "mopipi" in url2: + LogicLinkkf.referer = url + data2 = LogicLinkkf.get_html(url2) + # logger.info(data2) + match = re.compile(r"src\=\"(?Phttp.*?\.mp4)").search(data2) + video_url = match.group("video_url") + + match = re.compile(r"src\=\"(?Phttp.*?.vtt)").search(data2) + logger.info("match group: %s", match.group("video_url")) + vtt_url = match.group("vtt_url") + + # logger.info("download url : %s , url3 : %s" % (url, url3)) + + else: + logger.error("새로운 유형의 url 발생! %s %s" % (url, url2)) + except Exception as e: + logger.error("Exception:%s", e) + logger.error(traceback.format_exc()) + + return [video_url, referer_url, vtt_url] + + @staticmethod + def get_html_episode_content(url: str) -> str: + if url.startswith("http"): + html_data = LogicLinkkf.get_html(url) + else: + url = f"https://linkkf.app{url}" + + logger.info("get_video_url(): url: %s" % url) + data = LogicLinkkf.get_html(url) + + tree = html.fromstring(data) + + tree = html.fromstring(data) + + pattern = re.compile("var player_data=(.*)") + + js_scripts = tree.xpath("//script") + + iframe_info = None + index = 0 + + for js_script in js_scripts: + + # print(f"{index}.. {js_script.text_content()}") + if pattern.match(js_script.text_content()): + # logger.debug("match::::") + match_data = pattern.match(js_script.text_content()) + iframe_info = json.loads( + match_data.groups()[0].replace("path:", '"path":') + ) + # logger.debug(f"iframe_info:: {iframe_info}") + + index += 1 + + ################################################## + # iframe url:: https://s2.ani1c12.top/player/index.php?data='+player_data.url+' + #################################################### + + url = f'https://s2.ani1c12.top/player/index.php?data={iframe_info["url"]}' + html_data = LogicLinkkf.get_html(url) + + return html_data + def get_anime_info(self, cate, page): try: if cate == "ing": @@ -586,10 +997,230 @@ class LogicLinkkf(PluginModuleBase): ret = "%s.720p-SA.mp4" % maintitle return Util.change_text_for_use_filename(ret) + except Exception as e: + logger.error(f"Exception: {str(e)}") + logger.error(traceback.format_exc()) + + def add(self, episode_info): + if self.is_exist(episode_info): + return "queue_exits" + else: + + db_entity = ModelLinkkfItem.get_by_linkkf_id(episode_info["_id"]) + + logger.debug("db_entity:::> %s", db_entity) + # logger.debug("db_entity.status ::: %s", db_entity.status) + if db_entity is None: + entity = LinkkfQueueEntity(P, self, episode_info) + logger.debug("entity:::> %s", entity.as_dict()) + ModelLinkkfItem.append(entity.as_dict()) + # # logger.debug("entity:: type >> %s", type(entity)) + # + + self.queue.add_queue(entity) + # self.download_queue.add_queue(entity) + + # P.logger.debug(F.config['path_data']) + # P.logger.debug(self.headers) + + # filename = os.path.basename(entity.filepath) + # ffmpeg = SupportFfmpeg(entity.url, entity.filename, callback_function=self.callback_function, + # max_pf_count=0, + # save_path=entity.savepath, timeout_minute=60, headers=self.headers) + # ret = {'ret': 'success'} + # ret['json'] = ffmpeg.start() + return "enqueue_db_append" + elif db_entity.status != "completed": + entity = LinkkfQueueEntity(P, self, episode_info) + + logger.debug("entity:::> %s", entity.as_dict()) + + # P.logger.debug(F.config['path_data']) + # P.logger.debug(self.headers) + + filename = os.path.basename(entity.filepath) + ffmpeg = SupportFfmpeg(entity.url, entity.filename, callback_function=self.callback_function, + max_pf_count=0, save_path=entity.savepath, timeout_minute=60, + headers=self.headers) + ret = {'ret': 'success'} + ret['json'] = ffmpeg.start() + + # self.queue.add_queue(entity) + return "enqueue_db_exist" + else: + return "db_completed" + + # def is_exist(self, info): + # print(self.download_queue.entity_list) + # for en in self.download_queue.entity_list: + # if en.info["_id"] == info["_id"]: + # return True + + def is_exist(self, info): + for _ in self.queue.entity_list: + if _.info["_id"] == info["_id"]: + return True + return False + + def plugin_load(self): + try: + logger.debug("%s plugin_load", P.package_name) + # old version + self.queue = FfmpegQueue( + P, P.ModelSetting.get_int("ohli24_max_ffmpeg_process_count") + ) + self.current_data = None + self.queue.queue_start() + + # new version Todo: + # if self.download_queue is None: + # self.download_queue = queue.Queue() + # + # if self.download_thread is None: + # self.download_thread = threading.Thread(target=self.download_thread_function, args=()) + # self.download_thread.daemon = True + # self.download_thread.start() + except Exception as e: logger.error("Exception:%s", e) logger.error(traceback.format_exc()) + def plugin_unload(self): + pass + + def download_thread_function(self): + while True: + try: + while True: + logger.debug(self.current_download_count) + if self.current_download_count < P.ModelSetting.get_int(f"{self.name}_max_download_count"): + break + time.sleep(5) + + db_item = self.download_queue.get() + if db_item.status == "CANCEL": + self.download_queue.task_done() + continue + if db_item is None: + self.download_queue.task_done() + continue + + + except Exception as e: + logger.error(f'Exception: {str(e)}') + logger.error(traceback.format_exc()) + + +class LinkkfQueueEntity(FfmpegQueueEntity): + def __init__(self, P, module_logic, info): + super(LinkkfQueueEntity, self).__init__(P, module_logic, info) + self._vi = None + self.url = None + self.epi_queue = None + self.filepath = None + self.savepath = None + self.quality = None + self.filename = None + self.vtt = None + self.season = 1 + self.content_title = None + self.srt_url = None + self.headers = None + # Todo::: 임시 주석 처리 + self.make_episode_info() + + def refresh_status(self): + self.module_logic.socketio_callback("status", self.as_dict()) + + def info_dict(self, tmp): + # logger.debug('self.info::> %s', self.info) + for key, value in self.info.items(): + tmp[key] = value + tmp["vtt"] = self.vtt + tmp["season"] = self.season + tmp["content_title"] = self.content_title + tmp["linkkf_info"] = self.info + tmp["epi_queue"] = self.epi_queue + return tmp + + def make_episode_info(self): + url2s = [] + url = None + + try: + data = LogicLinkkf.get_html_episode_content(self.url) + tree = html.fromstring(data) + + xpath_select_query = '//*[@id="body"]/div/span/center/select/option' + + if len(tree.xpath(xpath_select_query)) > 0: + # by k45734 + print("ok") + xpath_select_query = '//select[@class="switcher"]/option' + for tag in tree.xpath(xpath_select_query): + url2s2 = tag.attrib["value"] + if "k40chan" in url2s2: + pass + elif "ani1c12" in url2s2: + pass + else: + url2s.append(url2s2) + else: + print(":: else ::") + + tt = re.search(r"var player_data=(.*?)<", data, re.S) + json_string = tt.group(1) + tt2 = re.search(r'"url":"(.*?)"', json_string, re.S) + json_string2 = tt2.group(1) + ttt = "https://s2.ani1c12.top/player/index.php?data=" + json_string2 + response = LogicLinkkf.get_html(ttt) + tree = html.fromstring(response) + xpath_select_query = '//select[@id="server-list"]/option' + for tag in tree.xpath(xpath_select_query): + url2s2 = tag.attrib["value"] + # if 'k40chan' in url2s2: + # pass + # elif 'k39aha' in url2s2: + if "ds" in url2s2: + pass + else: + url2s.append(url2s2) + + # logger.info('dx: url', url) + logger.info("dx: urls2:: %s", url2s) + + video_url = None + referer_url = None # dx + + for url2 in url2s: + try: + if video_url is not None: + continue + # logger.debug(f"url: {url}, url2: {url2}") + ret = LogicLinkkf.get_video_url_from_url(url, url2) + logger.debug(f"ret::::> {ret}") + + if ret is not None: + video_url = ret + referer_url = url2 + except Exception as e: + logger.error("Exception:%s", e) + logger.error(traceback.format_exc()) + + # logger.info(video_url) + # return [video_url, referer_url] + return video_url + logger.info("dx: urls2:: %s", url2s) + + video_url = None + referer_url = None # dx + + + + except Exception as e: + logger.error(f"Exception: {str(e)}") + logger.error(traceback.format_exc()) + class ModelLinkkfItem(db.Model): __tablename__ = "{package_name}_linkkf_item".format(package_name=P.package_name) @@ -604,7 +1235,7 @@ class ModelLinkkfItem(db.Model): episode_no = db.Column(db.Integer) title = db.Column(db.String) episode_title = db.Column(db.String) - linkkf_va = db.Column(db.String) + # linkkf_va = db.Column(db.String) linkkf_vi = db.Column(db.String) linkkf_id = db.Column(db.String) quality = db.Column(db.String) @@ -640,3 +1271,30 @@ class ModelLinkkfItem(db.Model): @classmethod def get_by_id(cls, idx): return db.session.query(cls).filter_by(id=idx).first() + + @classmethod + def get_by_linkkf_id(cls, linkkf_id): + return db.session.query(cls).filter_by(linkkf_id=linkkf_id).first() + + @classmethod + def append(cls, q): + logger.debug(q) + item = ModelLinkkfItem() + item.content_code = q["program_code"] + item.season = q["season"] + item.episode_no = q["epi_queue"] + item.title = q["content_title"] + item.episode_title = q["title"] + # item.linkkf_va = q["va"] + item.linkkf_code = q["code"] + item.linkkf_id = q["_id"] + item.quality = q["quality"] + item.filepath = q["filepath"] + item.filename = q["filename"] + item.savepath = q["savepath"] + item.video_url = q["url"] + item.vtt_url = q["vtt"] + item.thumbnail = q["image"][0] + item.status = "wait" + item.linkkf_info = q["linkkf_info"] + item.save() diff --git a/mod_ohli24.py b/mod_ohli24.py index 390dde8..eb69559 100644 --- a/mod_ohli24.py +++ b/mod_ohli24.py @@ -27,7 +27,7 @@ from flask import request, render_template, jsonify from lxml import html from sqlalchemy import or_, desc -pkgs = ["bs4", "jsbeautifier", "aiohttp"] +pkgs = ["bs4", "jsbeautifier", "aiohttp", "lxml", "loguru"] for pkg in pkgs: try: importlib.import_module(pkg) @@ -52,38 +52,22 @@ from framework import F from plugin import ( PluginModuleBase ) -from .lib._ffmpeg_queue import FfmpegQueueEntity, FfmpegQueue +from .lib.ffmpeg_queue import FfmpegQueueEntity, FfmpegQueue from support.expand.ffmpeg import SupportFfmpeg from .lib.util import Util +# from support_site import SupportKakaotv + from .setup import * logger = P.logger print('*=' * 50) +name = 'ohli24' class LogicOhli24(PluginModuleBase): - db_default = { - "ohli24_db_version": "1", - "ohli24_url": "https://ohli24.net", - "ohli24_download_path": os.path.join(path_data, P.package_name, "ohli24"), - "ohli24_auto_make_folder": "True", - "ohli24_auto_make_season_folder": "True", - "ohli24_finished_insert": "[완결]", - "ohli24_max_ffmpeg_process_count": "1", - "ohli24_order_desc": "False", - "ohli24_auto_start": "False", - "ohli24_interval": "* 5 * * *", - "ohli24_auto_mode_all": "False", - "ohli24_auto_code_list": "all", - "ohli24_current_code": "", - "ohli24_uncompleted_auto_enqueue": "False", - "ohli24_image_url_prefix_series": "https://www.jetcloud.cc/series/", - "ohli24_image_url_prefix_episode": "https://www.jetcloud-list.cc/thumbnail/", - "ohli24_discord_notify": "True", - } current_headers = None current_data = None @@ -104,9 +88,34 @@ class LogicOhli24(PluginModuleBase): "like Gecko) Chrome/96.0.4664.110 Whale/3.12.129.46 Safari/537.36" } + download_queue = None + download_thread = None + current_download_count = 0 + def __init__(self, P): super(LogicOhli24, self).__init__(P, "setting", scheduler_desc="ohli24 자동 다운로드") - self.name = "ohli24" + self.name = name + + self.db_default = { + "ohli24_db_version": "1", + "ohli24_url": "https://ohli24.net", + "ohli24_download_path": os.path.join(path_data, P.package_name, "ohli24"), + "ohli24_auto_make_folder": "True", + f"{self.name}_recent_code": "", + "ohli24_auto_make_season_folder": "True", + "ohli24_finished_insert": "[완결]", + "ohli24_max_ffmpeg_process_count": "1", + "ohli24_order_desc": "False", + "ohli24_auto_start": "False", + "ohli24_interval": "* 5 * * *", + "ohli24_auto_mode_all": "False", + "ohli24_auto_code_list": "all", + "ohli24_current_code": "", + "ohli24_uncompleted_auto_enqueue": "False", + "ohli24_image_url_prefix_series": "https://www.jetcloud.cc/series/", + "ohli24_image_url_prefix_episode": "https://www.jetcloud-list.cc/thumbnail/", + "ohli24_discord_notify": "True", + } self.queue = None # default_route_socketio(P, self) default_route_socketio_module(self, attach='/search') @@ -143,7 +152,6 @@ class LogicOhli24(PluginModuleBase): # @staticmethod def process_ajax(self, sub, req): - try: data = [] cate = request.form.get("type", None) @@ -156,6 +164,7 @@ class LogicOhli24(PluginModuleBase): bo_table = request.form.get("bo_table", None) P.ModelSetting.set("ohli24_current_code", code) data = self.get_series_info(code, wr_id, bo_table) + P.ModelSetting.set(f"{self.name}_recent_code", code) self.current_data = data return jsonify({"ret": "success", "data": data, "code": code}) elif sub == "anime_list": @@ -194,8 +203,31 @@ class LogicOhli24(PluginModuleBase): logger.info(f"info:: {info}") ret["ret"] = self.add(info) return jsonify(ret) + + # todo: new version + # info = json.loads(request.form["data"]) + # logger.info(info) + # logger.info(self.current_data) + # # 1. db 조회 + # db_item = ModelOhli24Program.get(info['_id']) + # logger.debug(db_item) + # + # if db_item is not None: + # print(f"db_item is not None") + # pass + # else: + # if db_item == None: + # db_item = ModelOhli24Program(info['_id'], self.get_episode(info['_id'])) + # db_item.save() + + + + elif sub == "entity_list": return jsonify(self.queue.get_entity_list()) + elif sub == "queue_list": + print(sub) + return {"test"} elif sub == "queue_command": ret = self.queue.command( req.form["command"], int(req.form["entity_id"]) @@ -248,6 +280,43 @@ class LogicOhli24(PluginModuleBase): P.logger.error(f"Exception: {e}") P.logger.error(traceback.format_exc()) + def get_episode(self, clip_id): + for _ in self.current_data["episode"]: + if _['title'] == clip_id: + return _ + + def process_command(self, command, arg1, arg2, arg3, req): + ret = {'ret': 'success'} + logger.debug('queue_list') + if command == 'queue_list': + logger.debug(f"self.queue.get_entity_list():: {self.queue.get_entity_list()}") + ret = [x for x in self.queue.get_entity_list()] + + return ret + elif command == 'download_program': + _pass = arg2 + db_item = ModelOhli24Program.get(arg1) + if _pass == 'false' and db_item != None: + ret['ret'] = 'warning' + ret['msg'] = '이미 DB에 있는 항목 입니다.' + elif _pass == 'true' and db_item != None and ModelOhli24Program.get_by_id_in_queue(db_item.id) != None: + ret['ret'] = 'warning' + ret['msg'] = '이미 큐에 있는 항목 입니다.' + else: + if db_item == None: + db_item = ModelOhli24Program(arg1, self.get_episode(arg1)) + db_item.save() + db_item.init_for_queue() + self.download_queue.put(db_item) + ret['msg'] = '다운로드를 추가 하였습니다.' + + elif command == 'list': + ret = [] + for ins in SupportFfmpeg.get_list(): + ret.append(ins.get_data()) + + return jsonify(ret) + @staticmethod def add_whitelist(*args): ret = {} @@ -295,7 +364,7 @@ class LogicOhli24(PluginModuleBase): ret["ret"] = False ret["log"] = "이미 추가되어 있습니다." except Exception as e: - logger.error("Exception:%s", e) + logger.error(f"Exception: {str(e)}") logger.error(traceback.format_exc()) ret["ret"] = False ret["log"] = str(e) @@ -319,9 +388,9 @@ class LogicOhli24(PluginModuleBase): week = ["월요일", "화요일", "수요일", "목요일", "금요일", "토요일", "일요일"] today = date.today() - print(today) - print() - print(today.weekday()) + # print(today) + # print() + # print(today.weekday()) url = f'{P.ModelSetting.get("ohli24_url")}/bbs/board.php?bo_table=ing&sca={week[today.weekday()]}' @@ -362,7 +431,7 @@ class LogicOhli24(PluginModuleBase): self.socketio_callback("list_refresh", "") # logger.debug(f"data: {data}") # self.current_data = data - # db에서 다운로드 완료 유무 체크 + # db 에서 다운로드 완료 유무 체크 @staticmethod async def get_data(url) -> str: @@ -675,15 +744,16 @@ class LogicOhli24(PluginModuleBase): # @staticmethod def plugin_load(self): try: - # ffmpeg_modelsetting = get_model_setting("ffmpeg", logger) - # SupportFfmpeg.initialize(P.ModelSetting.get('ffmpeg_path'), os.path.join(F.config['path_data'], 'tmp'), - # self.callback_function, P.ModelSetting.get_int('max_pf_count')) - # P.logger.debug(ffmpeg_modelsetting.get('ffmpeg_path')) + P.logger.debug(F.config['path_data']) # SupportFfmpeg.initialize(ffmpeg_modelsetting.get('ffmpeg_path'), os.path.join(F.config['path_data'], 'tmp'), # self.callback_function, ffmpeg_modelsetting.get_int('max_pf_count')) + # plugin loading download_queue 가 없으면 생성 + if self.download_queue is None: + self.download_queue = queue.Queue() + SupportFfmpeg.initialize("ffmpeg", os.path.join(F.config['path_data'], 'tmp'), self.callback_function, 1) @@ -692,7 +762,7 @@ class LogicOhli24(PluginModuleBase): P, P.ModelSetting.get_int("ohli24_max_ffmpeg_process_count") ) self.current_data = None - self.queue.queue_start() + # self.queue.queue_start() except Exception as e: logger.error("Exception:%s", e) @@ -747,7 +817,9 @@ class LogicOhli24(PluginModuleBase): return "queue_exist" else: db_entity = ModelOhli24Item.get_by_ohli24_id(episode_info["_id"]) - # logger.debug("db_entity:::> %s", db_entity) + + logger.debug("db_entity:::> %s", db_entity) + # logger.debug("db_entity.status ::: %s", db_entity.status) if db_entity is None: entity = Ohli24QueueEntity(P, self, episode_info) logger.debug("entity:::> %s", entity.as_dict()) @@ -771,8 +843,8 @@ class LogicOhli24(PluginModuleBase): logger.debug("entity:::> %s", entity.as_dict()) - P.logger.debug(F.config['path_data']) - P.logger.debug(self.headers) + # P.logger.debug(F.config['path_data']) + # P.logger.debug(self.headers) filename = os.path.basename(entity.filepath) ffmpeg = SupportFfmpeg(entity.url, entity.filename, callback_function=self.callback_function, @@ -787,10 +859,11 @@ class LogicOhli24(PluginModuleBase): return "db_completed" def is_exist(self, info): - # for en in self.queue.entity_list: - # if en.info["_id"] == info["_id"]: - # return True - return False + print(self.queue.entity_list) + for en in self.queue.entity_list: + if en.info["_id"] == info["_id"]: + return True + # return False def callback_function(self, **args): refresh_type = None @@ -1071,8 +1144,6 @@ class Ohli24QueueEntity(FfmpegQueueEntity): self.savepath = P.ModelSetting.get("ohli24_download_path") logger.info(f"self.savepath::> {self.savepath}") - # TODO: 완결 처리 - if P.ModelSetting.get_bool("ohli24_auto_make_folder"): if self.info["day"].find("완결") != -1: folder_name = "%s %s" % ( @@ -1097,7 +1168,8 @@ class Ohli24QueueEntity(FfmpegQueueEntity): self.savepath, self.filename.replace(".mp4", ".ko.srt") ) - if self.srt_url is not None and not os.path.exists(srt_filepath): + if self.srt_url is not None and not os.path.exists(srt_filepath) and not self.srt_url.split("/")[ + -1] == 'thumbnails.vtt': if requests.get(self.srt_url, headers=headers).status_code == 200: srt_data = requests.get(self.srt_url, headers=headers).text Util.write_file(srt_data, srt_filepath) @@ -1235,3 +1307,86 @@ class ModelOhli24Item(db.Model): item.status = "wait" item.ohli24_info = q["ohli24_info"] item.save() + + +class ModelOhli24Program(ModelBase): + P = P + __tablename__ = f'{P.package_name}_{name}_program' + __table_args__ = {'mysql_collate': 'utf8_general_ci'} + __bind_key__ = P.package_name + + id = db.Column(db.Integer, primary_key=True) + created_time = db.Column(db.DateTime, nullable=False) + completed_time = db.Column(db.DateTime) + completed = db.Column(db.Boolean) + + clip_id = db.Column(db.String) + info = db.Column(db.String) + status = db.Column(db.String) + call = db.Column(db.String) + queue_list = [] + + def __init__(self, clip_id, info, call='user'): + self.clip_id = clip_id + self.info = info + self.completed = False + self.created_time = datetime.now() + self.status = "READY" + self.call = call + + def init_for_queue(self): + self.status = "READY" + self.queue_list.append(self) + + @classmethod + def get(cls, clip_id): + with F.app.app_context(): + return db.session.query(cls).filter_by( + clip_id=clip_id, + ).order_by(desc(cls.id)).first() + + @classmethod + def is_duplicate(cls, clip_id): + return (cls.get(clip_id) != None) + + # 오버라이딩 + @classmethod + def make_query(cls, req, order='desc', search='', option1='all', option2='all'): + with F.app.app_context(): + query = F.db.session.query(cls) + # query = cls.make_query_search(query, search, cls.program_title) + query = query.filter(cls.info['channel_name'].like('%' + search + '%')) + if option1 == 'completed': + query = query.filter_by(completed=True) + elif option1 == 'incompleted': + query = query.filter_by(completed=False) + elif option1 == 'auto': + query = query.filter_by(call="user") + + if order == 'desc': + query = query.order_by(desc(cls.id)) + else: + query = query.order_by(cls.id) + return query + + @classmethod + def remove_all(cls, is_completed=True): # to remove_all(True/False) + with F.app.app_context(): + count = db.session.query(cls).filter_by(completed=is_completed).delete() + db.session.commit() + return count + + @classmethod + def get_failed(cls): + with F.app.app_context(): + return db.session.query(cls).filter_by( + completed=False + ).all() + + ### only for queue + @classmethod + def get_by_id_in_queue(cls, id): + for _ in cls.queue_list: + if _.id == int(id): + return _ + ### only for queue END diff --git a/templates/anime_downloader_anilife_queue.html b/templates/anime_downloader_anilife_queue.html index 6ec4add..29b84d0 100644 --- a/templates/anime_downloader_anilife_queue.html +++ b/templates/anime_downloader_anilife_queue.html @@ -1,130 +1,175 @@ {% extends "base.html" %} {% block content %} -
- {{ macros.m_button_group([['reset_btn', '초기화'], ['delete_completed_btn', '완료 목록 삭제'], ['go_ffmpeg_btn', 'Go FFMPEG']])}} - {{ macros.m_row_start('0') }} - {{ macros.m_row_end() }} - {{ macros.m_hr_head_top() }} - {{ macros.m_row_start('0') }} - {{ macros.m_col(1, macros.m_strong('Idx')) }} - {{ macros.m_col(2, macros.m_strong('CreatedTime')) }} - {{ macros.m_col(4, macros.m_strong('Filename')) }} - {{ macros.m_col(3, macros.m_strong('Status')) }} - {{ macros.m_col(2, macros.m_strong('Action')) }} - {{ macros.m_row_end() }} - {{ macros.m_hr_head_bottom() }} -
-
+ + + + + + + + + + + + + + + + + + +
IDXPlugin시작시간파일명상태진행률길이PF배속진행시간Action
-{% endblock %} \ No newline at end of file + +$("body").on('click', '#stop_btn', function(e){ + e.stopPropagation(); + e.preventDefault(); + globalSendCommand('stop', $(this).data('idx'), null, null, function(ret){ + refresh_item(ret.data); + }); +}); + + +function refresh_item(data) { + $('#tr1_'+data.idx).html(make_item1(data)); + $('#collapse_'+data.idx).html(make_item2(data)); +} + +function make_item(data) { + str = ''; + str += make_item1(data); + str += ''; + str += ''; + str += make_item2(data); + str += ''; + return str; +} + +function make_item1(data) { + //console.log(data); + str = ''; + str += ''+ data.idx + ''; + str += ''+ data.callback_id + ''; + str += ''+ data.start_time + ''; + str += ''+ data.filename + ''; + str += ''+ data.status_kor + ''; + var visi = 'hidden'; + if (parseInt(data.percent) > 0) { + visi = 'visible'; + } + str += '
'+data.percent +'%
'; + str += ''+ data.duration_str + ''; + str += ''+ data.current_pf_count + ''; + str += ''+ data.current_speed + ''; + str += ''+ data.download_time + ''; + str += ''; + if (data.status_str == 'DOWNLOADING') { + str += j_button('stop_btn', '중지', {'idx':data.idx}, 'danger', false, false); + } + str += '' + return str; +} + +function make_item2(data) { + str = ''; + str += ''; + str += '
'; + str += get_detail(data); + str += '
'; + str += ''; + return str +} + + +function get_detail(data) { + var str = j_row_info('URL', data.url); + str += j_row_info('임시경로', data.temp_fullpath); + str += j_row_info('저장경로', data.save_fullpath); + str += j_row_info('진행률(current/total)', data.percent+ '% (' + data.current_duration + ' / ' + data.duration + ')'); + str += j_row_info('현재 비트레이트', data.current_bitrate); + str += j_row_info('종료시간', data.end_time); + str += j_row_info('허용 Packet Fail 수', data.max_pf_count); + str += j_row_info('파일 Exist', data.exist); + if (data.status_str == 'COMPLETED') { + str += j_row_info('파일 크기', data.filesize_str); + str += j_row_info('다운 속도', data.download_speed); + } + return str; +} + +function button_html(data) { + //console.log(data) + str = ''; + if (data.status_str == 'DOWNLOADING') { + str = j_button('stop_btn', '중지', {'idx':data.idx}, 'danger', false, false); + } + $("#button_" + data.idx).html(str); +} + +function status_html(data) { + var progress = document.getElementById("progress_" + data.idx); + progress.style.width = data.percent+ '%'; + progress.innerHTML = data.percent+ '%'; + progress.style.visibility = 'visible'; + document.getElementById("status_" + data.idx).innerHTML = data.status_kor; + document.getElementById("current_pf_count_" + data.idx).innerHTML = data.current_pf_count; + document.getElementById("current_speed_" + data.idx).innerHTML = data.current_speed; + document.getElementById("download_time_" + data.idx).innerHTML = data.download_time; + document.getElementById("detail_" + data.idx).innerHTML = get_detail(data); +} + + +{% endblock %} diff --git a/templates/anime_downloader_anilife_search.html b/templates/anime_downloader_anilife_search.html index 19b61ee..9f32e92 100644 --- a/templates/anime_downloader_anilife_search.html +++ b/templates/anime_downloader_anilife_search.html @@ -57,11 +57,7 @@
-
- -
+
@@ -132,35 +128,44 @@ } $.ajax({ - url: url, - type: "POST", - data: data, - cache: false, - dataType: "json", - success: (ret) => { - current_screen_movie_data = ret - console.log('ret::>', ret) - - if (current_screen_movie_data !== '') { - if (type === "ing") { - make_airing_list(ret.data, page) - observer.observe(); - } else if (type === "fin") { - make_screen_movie_list(ret.data, page) - observer.observe(); - } else if (type === "theater") { - make_screen_movie_list(ret.data, page) - observer.observe(); - } else { - make_screen_movie_list(ret.data, page) + url: url, + type: "POST", + data: data, + cache: false, + dataType: "json", + success: (ret) => { + if (ret.ret === "error") { + $.notify("분석 실패
" + ret.log, { + type: "warning", + }); + return false; } - div_visible = true - console.log(div_visible) + + + current_screen_movie_data = ret + console.log('ret::>', ret) + + if (current_screen_movie_data !== '') { + if (type === "ing") { + make_airing_list(ret.data, page) + observer.observe(); + } else if (type === "fin") { + make_screen_movie_list(ret.data, page) + observer.observe(); + } else if (type === "theater") { + make_screen_movie_list(ret.data, page) + observer.observe(); + } else { + make_screen_movie_list(ret.data, page) + } + div_visible = true + console.log(div_visible) + } + dismissLoadingScreen() + next_page = page + 1 } - dismissLoadingScreen() - next_page = page + 1 } - }) + ) } function make_airing_list(data, page) { diff --git a/templates/anime_downloader_linkkf_queue.html b/templates/anime_downloader_linkkf_queue.html index e9917cf..29b84d0 100644 --- a/templates/anime_downloader_linkkf_queue.html +++ b/templates/anime_downloader_linkkf_queue.html @@ -1,131 +1,175 @@ {% extends "base.html" %} {% block content %} -
- {{ macros.m_button_group([['reset_btn', '초기화'], ['delete_completed_btn', '완료 목록 삭제'], ['go_ffmpeg_btn', 'Go FFMPEG']])}} - {{ macros.m_row_start('0') }} - {{ macros.m_row_end() }} - {{ macros.m_hr_head_top() }} - {{ macros.m_row_start('0') }} - {{ macros.m_col(1, macros.m_strong('Idx')) }} - {{ macros.m_col(2, macros.m_strong('CreatedTime')) }} - {{ macros.m_col(4, macros.m_strong('Filename')) }} - {{ macros.m_col(3, macros.m_strong('Status')) }} - {{ macros.m_col(2, macros.m_strong('Action')) }} - {{ macros.m_row_end() }} - {{ macros.m_hr_head_bottom() }} -
-
+ + + + + + + + + + + + + + + + + + +
IDXPlugin시작시간파일명상태진행률길이PF배속진행시간Action
-{% endblock %} \ No newline at end of file + +$("body").on('click', '#stop_btn', function(e){ + e.stopPropagation(); + e.preventDefault(); + globalSendCommand('stop', $(this).data('idx'), null, null, function(ret){ + refresh_item(ret.data); + }); +}); + + +function refresh_item(data) { + $('#tr1_'+data.idx).html(make_item1(data)); + $('#collapse_'+data.idx).html(make_item2(data)); +} + +function make_item(data) { + str = ''; + str += make_item1(data); + str += ''; + str += ''; + str += make_item2(data); + str += ''; + return str; +} + +function make_item1(data) { + //console.log(data); + str = ''; + str += ''+ data.idx + ''; + str += ''+ data.callback_id + ''; + str += ''+ data.start_time + ''; + str += ''+ data.filename + ''; + str += ''+ data.status_kor + ''; + var visi = 'hidden'; + if (parseInt(data.percent) > 0) { + visi = 'visible'; + } + str += '
'+data.percent +'%
'; + str += ''+ data.duration_str + ''; + str += ''+ data.current_pf_count + ''; + str += ''+ data.current_speed + ''; + str += ''+ data.download_time + ''; + str += ''; + if (data.status_str == 'DOWNLOADING') { + str += j_button('stop_btn', '중지', {'idx':data.idx}, 'danger', false, false); + } + str += '' + return str; +} + +function make_item2(data) { + str = ''; + str += ''; + str += '
'; + str += get_detail(data); + str += '
'; + str += ''; + return str +} + + +function get_detail(data) { + var str = j_row_info('URL', data.url); + str += j_row_info('임시경로', data.temp_fullpath); + str += j_row_info('저장경로', data.save_fullpath); + str += j_row_info('진행률(current/total)', data.percent+ '% (' + data.current_duration + ' / ' + data.duration + ')'); + str += j_row_info('현재 비트레이트', data.current_bitrate); + str += j_row_info('종료시간', data.end_time); + str += j_row_info('허용 Packet Fail 수', data.max_pf_count); + str += j_row_info('파일 Exist', data.exist); + if (data.status_str == 'COMPLETED') { + str += j_row_info('파일 크기', data.filesize_str); + str += j_row_info('다운 속도', data.download_speed); + } + return str; +} + +function button_html(data) { + //console.log(data) + str = ''; + if (data.status_str == 'DOWNLOADING') { + str = j_button('stop_btn', '중지', {'idx':data.idx}, 'danger', false, false); + } + $("#button_" + data.idx).html(str); +} + +function status_html(data) { + var progress = document.getElementById("progress_" + data.idx); + progress.style.width = data.percent+ '%'; + progress.innerHTML = data.percent+ '%'; + progress.style.visibility = 'visible'; + document.getElementById("status_" + data.idx).innerHTML = data.status_kor; + document.getElementById("current_pf_count_" + data.idx).innerHTML = data.current_pf_count; + document.getElementById("current_speed_" + data.idx).innerHTML = data.current_speed; + document.getElementById("download_time_" + data.idx).innerHTML = data.download_time; + document.getElementById("detail_" + data.idx).innerHTML = get_detail(data); +} + + +{% endblock %} diff --git a/templates/anime_downloader_linkkf_request.html b/templates/anime_downloader_linkkf_request.html index c641c8a..0eece75 100644 --- a/templates/anime_downloader_linkkf_request.html +++ b/templates/anime_downloader_linkkf_request.html @@ -1,4 +1,5 @@ {% extends "base.html" %} {% block content %} +
@@ -30,6 +31,7 @@
+
@@ -171,6 +173,14 @@ tmp += m_button("add_queue_btn", "다운로드 추가", [ {key: "idx", value: i}, ]); + tmp += j_button('insert_download_btn', '다운로드 추가', { + code: data.episode[i]._id, + }); + tmp += j_button( + 'force_insert_download_btn', + '다운로드 추가 (DB무시)', + {code: data.episode[i]._id} + ); // tmp += ''; tmp += "
"; str += m_col(12, tmp); @@ -316,6 +326,20 @@ }); +
+
+
+
+
+
+
+
+
+ + + +
+
+ {{ macros.setting_input_text_and_buttons('code', '작품 Code', [['analysis_btn', '분석'], + ['go_ohli24_btn', 'Go OHLI24']], desc='예) "https://ohli24.net/c/녹을 먹는 비스코" 이나 "녹을 + 먹는 비스코"') }} +
+
+
+
+
+ + + + + {% endblock %} diff --git a/templates/anime_downloader_ohli24_search.html b/templates/anime_downloader_ohli24_search.html index de21fc6..fa2332b 100644 --- a/templates/anime_downloader_ohli24_search.html +++ b/templates/anime_downloader_ohli24_search.html @@ -1,5 +1,5 @@ {% extends "base.html" %} {% block content %} - +
@@ -53,11 +53,7 @@
-
- -
+
@@ -139,24 +135,24 @@ cache: false, dataType: "json", success: (ret) => { - current_screen_movie_data = ret + let current_screen_movie_data = ret console.log('ret::>', ret) if (current_screen_movie_data !== '') { if (type === "ing") { make_airing_list(ret.data, page) - observer.observe(); + {#observer.observe();#} } else if (type === "fin") { make_screen_movie_list(ret.data, page) - observer.observe(); + {#observer.observe();#} } else if (type === "theater") { make_screen_movie_list(ret.data, page) - observer.observe(); + {#observer.observe();#} } else { make_screen_movie_list(ret.data, page) } - div_visible = true - console.log(div_visible) + {#div_visible = true#} + {#console.log(div_visible)#} } next_page = page + 1 } @@ -341,7 +337,6 @@ // } $("#input_search").keydown(function (key) { if (key.keyCode === 13) { - // alert("엔터키를 눌렀습니다."); $("#btn_search").trigger("click"); } })