From 9e0ef5f120b1cdb33d0a891f93dc3f450a1c11a0 Mon Sep 17 00:00:00 2001 From: projectdx Date: Wed, 7 Jan 2026 17:14:25 +0900 Subject: [PATCH] v0.6.15: Non-blocking navigation for Zendriver Daemon (Docker 17s fix) --- README.md | 7 +++++ info.yaml | 2 +- lib/zendriver_daemon.py | 70 +++++++++++++++++++++++++++-------------- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index a187c8f..e12774c 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,13 @@ ## ๐Ÿ“ ๋ณ€๊ฒฝ ์ด๋ ฅ (Changelog) +### v0.6.15 (2026-01-07) +- **Zendriver Daemon ๋น„๋™๊ธฐ ๋„ค๋น„๊ฒŒ์ด์…˜ ์ตœ์ ํ™”**: + - `browser.get(url)` ๋Œ€๊ธฐ ์‹œ๊ฐ„์œผ๋กœ ์ธํ•œ 17์ดˆ ์ง€์—ฐ ํ•ด๊ฒฐ + - ๋„ค๋น„๊ฒŒ์ด์…˜ ์ „ ๋ฆฌ์†Œ์Šค ๋ธ”๋กœํ‚น ์„ค์ • (about:blank์—์„œ ๋ฏธ๋ฆฌ ์ฐจ๋‹จ) + - `asyncio.create_task()`๋ฅผ ํ™œ์šฉํ•œ ๋น„๋™๊ธฐ ๋„ค๋น„๊ฒŒ์ด์…˜ + ๋ณ‘๋ ฌ DOM ํด๋ง + - ๋ฆฌ์ŠคํŠธ/์—ํ”ผ์†Œ๋“œ ํŽ˜์ด์ง€ ๋งˆ์ปค ๋ฐœ๊ฒฌ ์ฆ‰์‹œ ์กฐ๊ธฐ ๋ฐ˜ํ™˜ โ†’ ์˜ˆ์ƒ ์†๋„ 1~3์ดˆ + ### v0.6.14 (2026-01-07) - **Ohli24 Docker ์„ฑ๋Šฅ ๊ณ ์†ํ™”**: - Zendriver Daemon์— ๋ฆฌ๋ˆ…์Šค/๋„์ปค ์ „์šฉ ์ตœ์ ํ™” ํ”Œ๋ž˜๊ทธ ์ถ”๊ฐ€ (`--no-zygote`, `--disable-dev-shm-usage`, `--disable-features=IsolateOrigins,site-per-process` ๋“ฑ) diff --git a/info.yaml b/info.yaml index 27e915c..a2f8c06 100644 --- a/info.yaml +++ b/info.yaml @@ -1,5 +1,5 @@ title: "์• ๋‹ˆ ๋‹ค์šด๋กœ๋”" -version: "0.6.14" +version: "0.6.15" package_name: "anime_downloader" developer: "projectdx" description: "anime downloader" diff --git a/lib/zendriver_daemon.py b/lib/zendriver_daemon.py index 096e1c9..bd07388 100644 --- a/lib/zendriver_daemon.py +++ b/lib/zendriver_daemon.py @@ -273,48 +273,70 @@ async def fetch_with_browser(url: str, timeout: int = 30) -> Dict[str, Any]: log_debug(f"[ZendriverDaemon] Fetching URL: {url} (Init: {init_elapsed:.2f}s)") try: - nav_start = time.time() - # browser.get(url)์€ ์ƒˆ ํƒญ์„ ์—ด๊ฑฐ๋‚˜ ๊ธฐ์กด ํƒญ์„ ์‚ฌ์šฉํ•จ - page: Any = await browser.get(url) - nav_elapsed = time.time() - nav_start - - # ๋ฆฌ์†Œ์Šค ๋ธ”๋กœํ‚น (CDP ํ™œ์šฉ) - CSS, ํฐํŠธ, ์ด๋ฏธ์ง€ ๋“ฑ ์ฐจ๋‹จ์œผ๋กœ ์†๋„ ํ–ฅ์ƒ - block_start = time.time() + # ํƒญ ๋งŒ๋“ค๊ธฐ ๋ฐ ๋ฆฌ์†Œ์Šค ์ฐจ๋‹จ ๋ฏธ๋ฆฌ ์„ค์ • (๋„ค๋น„๊ฒŒ์ด์…˜ ์ „) + page: Any = await browser.get("about:blank") try: - await page.send(zd.cdp.network.set_blocked_urls(urls=[ - "*.jpg", "*.jpeg", "*.png", "*.gif", "*.svg", "*.webp", "*.ico", - "*.css", "*.woff", "*.woff2", "*.ttf", "*.eot", - "*ads*", "*google-analytics*", "*googletagmanager*", "*doubleclick*" - ])) - await page.send(zd.cdp.network.enable()) + await page.send(zd.cdp.network.set_blocked_urls(urls=[ + "*.jpg", "*.jpeg", "*.png", "*.gif", "*.svg", "*.webp", "*.ico", + "*.css", "*.woff", "*.woff2", "*.ttf", "*.eot", + "*ads*", "*google-analytics*", "*googletagmanager*", "*doubleclick*" + ])) + await page.send(zd.cdp.network.enable()) except Exception as e: - log_debug(f"[ZendriverDaemon] Resource blocking enable failed: {e}") - block_elapsed = time.time() - block_start + log_debug(f"[ZendriverDaemon] Pre-nav blocking failed: {e}") - # ํŽ˜์ด์ง€ ๋กœ๋“œ ๋Œ€๊ธฐ - ์ง€๋Šฅํ˜• ํด๋ง (์ตœ๋Œ€ 10์ดˆ) - # 1. ๋ฆฌ์ŠคํŠธ ํŽ˜์ด์ง€๋Š” ๋ฐ”๋กœ ๋ฐ˜ํ™˜, 2. ์—ํ”ผ์†Œ๋“œ ํŽ˜์ด์ง€๋Š” ํ”Œ๋ ˆ์ด์–ด ๋กœ๋”ฉ ๋Œ€๊ธฐ - max_wait = 10 - poll_interval = 0.1 # 0.2s -> 0.1s๋กœ ๋” ๋น ๋ฅด๊ฒŒ ์ฒดํฌ + nav_start = time.time() + # zendriver์˜ page.get(url)์€ ๊ธฐ๋ณธ์ ์œผ๋กœ ์™„๋ฃŒ๋ฅผ ๊ธฐ๋‹ค๋ฆผ. + # 17s ์ง€์—ฐ์„ ํ”ผํ•˜๊ธฐ ์œ„ํ•ด navigation ๋„์ค‘์—๋ผ๋„ polling์„ ์‹œ์ž‘ํ•ด์•ผ ํ•จ. + # asyncio.wait_for์™€ page.get()์„ ์กฐํ•ฉํ•˜์—ฌ ๊ฐ•์ œ ํƒ€์ž„์•„์›ƒ ์ฃผ๊ฑฐ๋‚˜, + # ๋‹จ์ˆœํžˆ page.get()์„ ํƒœ์Šคํฌ๋กœ ๋˜์ง€๊ณ  ๊ฐ์‹œ ์‹œ๋„ + + nav_task = asyncio.create_task(page.goto(url)) # goto is usually slightly more flexible + nav_elapsed = 0.0 + + # ํŽ˜์ด์ง€ ๋กœ๋“œ ๋Œ€๊ธฐ - ์ง€๋Šฅํ˜• ํด๋ง (navigation๊ณผ ๋ณ‘๋ ฌ ์‹คํ–‰) + max_wait = 15 + poll_interval = 0.2 waited = 0 html_content = "" + poll_start = time.time() while waited < max_wait: await asyncio.sleep(poll_interval) waited += poll_interval - html_content = await page.get_content() - # ๋ฆฌ์ŠคํŠธ ํŽ˜์ด์ง€ ๋งˆ์ปค ํ™•์ธ (๋ฐœ๊ฒฌ ์ฆ‰์‹œ ํƒˆ์ถœ) + try: + html_content = await page.get_content() + except: + continue # ์•„์ง ๋กœ๋“œ ์ค‘์ผ ์ˆ˜ ์žˆ์Œ + + # ๋ฆฌ์ŠคํŠธ ํŽ˜์ด์ง€ ๋งˆ์ปค ํ™•์ธ if "post-list" in html_content or "list-box" in html_content or "post-row" in html_content: - log_debug(f"[ZendriverDaemon] List page detected in {waited:.1f}s") + log_debug(f"[ZendriverDaemon] List page detected early in {waited:.1f}s") break - # cdndania/fireplayer iframe์ด ๋กœ๋“œ๋˜์—ˆ๋Š”์ง€ ํ™•์ธ (์—ํ”ผ์†Œ๋“œ ํŽ˜์ด์ง€) + # ์—ํ”ผ์†Œ๋“œ ํŽ˜์ด์ง€ if "cdndania" in html_content or "fireplayer" in html_content: - log_debug(f"[ZendriverDaemon] Player detected in {waited:.1f}s") + log_debug(f"[ZendriverDaemon] Player detected early in {waited:.1f}s") break + + # nav_task๊ฐ€ ์™„๋ฃŒ๋˜์—ˆ๋Š”์ง€ ํ™•์ธ (์ด๋ฏธ load ์™„๋ฃŒ๋œ ๊ฒฝ์šฐ) + if nav_task.done(): + try: + await nav_task # ์˜ˆ์™ธ ํ™•์ธ + log_debug(f"[ZendriverDaemon] Navigation task completed in {waited:.1f}s") + except Exception as nav_e: + log_debug(f"[ZendriverDaemon] Navigation task error: {nav_e}") + break + + if not nav_task.done(): + log_debug(f"[ZendriverDaemon] Navigation still in progress, but we have content (len: {len(html_content)})") + # nav_task.cancel() # ์ทจ์†Œํ• ์ง€ ๋ง์ง€๋Š” ๊ณ ๋ฏผ ํ•„์š” (๋ณดํ†ต ๋ƒ…๋‘ฌ๋„ ๋จ) poll_elapsed = time.time() - poll_start + nav_elapsed = time.time() - nav_start total_elapsed = time.time() - start_time + block_elapsed = 0.0 # Pre-nav blocking์— ํฌํ•จ๋จ if html_content and len(html_content) > 100: result.update({