fix: close Chromium tabs on crawler fetch error paths
new_page() opened a tab that was only closed on the happy path; a `?` on wait_for_nav / content() leaked it (chromiumoxide doesn't close on drop), so errored fetches accumulated tabs in the shared browser. Add a generic, unit-tested `close_after(close, body)` helper and wrap the three fetch sites (chapter content, session probe, source navigate) so the tab is closed on every exit path — including navigate's content()-read branch, which previously leaked. Bump to 0.124.5. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2
backend/Cargo.lock
generated
2
backend/Cargo.lock
generated
@@ -1558,7 +1558,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.124.4"
|
version = "0.124.5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"argon2",
|
"argon2",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mangalord"
|
name = "mangalord"
|
||||||
version = "0.124.4"
|
version = "0.124.5"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
default-run = "mangalord"
|
default-run = "mangalord"
|
||||||
|
|
||||||
|
|||||||
@@ -122,22 +122,31 @@ async fn fetch_chapter_html_once(
|
|||||||
.new_page(source_url)
|
.new_page(source_url)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("open chapter page {source_url}"))?;
|
.with_context(|| format!("open chapter page {source_url}"))?;
|
||||||
crate::crawler::nav::wait_for_nav(&page)
|
// Close the tab on every exit path — a `?` on wait_for_nav / content()
|
||||||
.await
|
// would otherwise leak it (chromiumoxide doesn't close on drop).
|
||||||
.context("wait for chapter nav")?;
|
let closer = page.clone();
|
||||||
// Best-effort wait for the reader marker — same partial-render
|
crate::crawler::nav::close_after(
|
||||||
// race that bit the chapter-list parser can hit here. Timeout is
|
async move {
|
||||||
// not an error; the chapter probe + parser sentinels still catch
|
closer.close().await.ok();
|
||||||
// real failures.
|
},
|
||||||
let _ = crate::crawler::nav::wait_for_selector(
|
async {
|
||||||
&page,
|
crate::crawler::nav::wait_for_nav(&page)
|
||||||
"a#pic_container",
|
.await
|
||||||
crate::crawler::nav::SELECTOR_TIMEOUT,
|
.context("wait for chapter nav")?;
|
||||||
|
// Best-effort wait for the reader marker — same partial-render
|
||||||
|
// race that bit the chapter-list parser can hit here. Timeout is
|
||||||
|
// not an error; the chapter probe + parser sentinels still catch
|
||||||
|
// real failures.
|
||||||
|
let _ = crate::crawler::nav::wait_for_selector(
|
||||||
|
&page,
|
||||||
|
"a#pic_container",
|
||||||
|
crate::crawler::nav::SELECTOR_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
page.content().await.context("read chapter html")
|
||||||
|
},
|
||||||
)
|
)
|
||||||
.await;
|
.await
|
||||||
let html = page.content().await.context("read chapter html")?;
|
|
||||||
page.close().await.ok();
|
|
||||||
Ok(html)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Pure-over-IO loop: fetch + classify, up to `max_attempts` total
|
/// Pure-over-IO loop: fetch + classify, up to `max_attempts` total
|
||||||
|
|||||||
@@ -85,6 +85,28 @@ pub async fn wait_for_selector(
|
|||||||
/// under a second.
|
/// under a second.
|
||||||
pub const SELECTOR_TIMEOUT: Duration = Duration::from_secs(10);
|
pub const SELECTOR_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
|
/// Run `body` to completion, then run `close` — on *every* exit path,
|
||||||
|
/// including an early `?` / `return` inside `body`. chromiumoxide's `Page`
|
||||||
|
/// does **not** close its CDP target on drop, so a fetch helper that opens a
|
||||||
|
/// page with `browser.new_page(...)` and then bails via `?` on a nav /
|
||||||
|
/// content-read error would leak a browser tab for the process's lifetime.
|
||||||
|
/// Wrapping the fallible work in `close_after` — with `close` built from a
|
||||||
|
/// clone of the page — guarantees the tab is closed regardless of how `body`
|
||||||
|
/// returns.
|
||||||
|
///
|
||||||
|
/// Both arguments are pre-built futures, so this stays generic over the
|
||||||
|
/// body's return type (the anyhow and `PageError` fetch paths both use it)
|
||||||
|
/// and references no browser types — which also lets it be unit-tested
|
||||||
|
/// without standing up a real `Page`.
|
||||||
|
pub(crate) async fn close_after<R>(
|
||||||
|
close: impl std::future::Future<Output = ()>,
|
||||||
|
body: impl std::future::Future<Output = R>,
|
||||||
|
) -> R {
|
||||||
|
let result = body.await;
|
||||||
|
close.await;
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
impl NavError {
|
impl NavError {
|
||||||
/// Does this navigation error indicate the underlying Chromium
|
/// Does this navigation error indicate the underlying Chromium
|
||||||
/// process has died or its CDP connection has dropped? Used by the
|
/// process has died or its CDP connection has dropped? Used by the
|
||||||
@@ -150,6 +172,41 @@ mod tests {
|
|||||||
assert!(result.is_err(), "expected Elapsed on a hung future");
|
assert!(result.is_err(), "expected Elapsed on a hung future");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn close_after_runs_close_and_returns_value_on_ok() {
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::sync::Arc;
|
||||||
|
let closed = Arc::new(AtomicBool::new(false));
|
||||||
|
let c = closed.clone();
|
||||||
|
let out: anyhow::Result<i32> = close_after(
|
||||||
|
async move { c.store(true, Ordering::SeqCst) },
|
||||||
|
async { Ok(7) },
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(out.unwrap(), 7);
|
||||||
|
assert!(closed.load(Ordering::SeqCst), "close must run on the happy path");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn close_after_runs_close_even_when_body_errs() {
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::sync::Arc;
|
||||||
|
let closed = Arc::new(AtomicBool::new(false));
|
||||||
|
let c = closed.clone();
|
||||||
|
// This is the leak the fix targets: the body bails before it could
|
||||||
|
// close the page itself, and `close_after` must still close it.
|
||||||
|
let out: anyhow::Result<()> =
|
||||||
|
close_after(async move { c.store(true, Ordering::SeqCst) }, async {
|
||||||
|
anyhow::bail!("nav failed")
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(out.is_err());
|
||||||
|
assert!(
|
||||||
|
closed.load(Ordering::SeqCst),
|
||||||
|
"the page must be closed even when the body returns Err"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn nav_error_timeout_message_includes_duration() {
|
fn nav_error_timeout_message_includes_duration() {
|
||||||
let e = NavError::Timeout(Duration::from_secs(30));
|
let e = NavError::Timeout(Duration::from_secs(30));
|
||||||
|
|||||||
@@ -295,21 +295,30 @@ async fn fetch_probe_html(browser: &Browser, probe_url: &str) -> anyhow::Result<
|
|||||||
.new_page(probe_url)
|
.new_page(probe_url)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("open probe page {probe_url}"))?;
|
.with_context(|| format!("open probe page {probe_url}"))?;
|
||||||
crate::crawler::nav::wait_for_nav(&page)
|
// Close the tab on every exit path — a `?` on wait_for_nav / content()
|
||||||
.await
|
// would otherwise leak it (chromiumoxide doesn't close on drop).
|
||||||
.context("wait for nav on probe")?;
|
let closer = page.clone();
|
||||||
// Best-effort wait for the layout marker. Timeout is fine — the
|
crate::crawler::nav::close_after(
|
||||||
// probe classifier handles a missing `#logo` as Transient anyway,
|
async move {
|
||||||
// and the verify loop retries on Transient.
|
closer.close().await.ok();
|
||||||
let _ = crate::crawler::nav::wait_for_selector(
|
},
|
||||||
&page,
|
async {
|
||||||
"#logo",
|
crate::crawler::nav::wait_for_nav(&page)
|
||||||
crate::crawler::nav::SELECTOR_TIMEOUT,
|
.await
|
||||||
|
.context("wait for nav on probe")?;
|
||||||
|
// Best-effort wait for the layout marker. Timeout is fine — the
|
||||||
|
// probe classifier handles a missing `#logo` as Transient anyway,
|
||||||
|
// and the verify loop retries on Transient.
|
||||||
|
let _ = crate::crawler::nav::wait_for_selector(
|
||||||
|
&page,
|
||||||
|
"#logo",
|
||||||
|
crate::crawler::nav::SELECTOR_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
page.content().await.context("read probe html")
|
||||||
|
},
|
||||||
)
|
)
|
||||||
.await;
|
.await
|
||||||
let html = page.content().await.context("read probe html")?;
|
|
||||||
page.close().await.ok();
|
|
||||||
Ok(html)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use super::{
|
|||||||
use crate::crawler::detect::{
|
use crate::crawler::detect::{
|
||||||
has_logo_sentinel, is_broken_page_body, retry_on_transient_with_hook, PageError,
|
has_logo_sentinel, is_broken_page_body, retry_on_transient_with_hook, PageError,
|
||||||
};
|
};
|
||||||
use crate::crawler::nav::{wait_for_nav, wait_for_selector, NavError, SELECTOR_TIMEOUT};
|
use crate::crawler::nav::{close_after, wait_for_nav, wait_for_selector, NavError, SELECTOR_TIMEOUT};
|
||||||
use crate::crawler::safety::ensure_public_target;
|
use crate::crawler::safety::ensure_public_target;
|
||||||
|
|
||||||
/// `sources.id` value for this Source impl. Exposed as a const so the
|
/// `sources.id` value for this Source impl. Exposed as a const so the
|
||||||
@@ -258,26 +258,34 @@ async fn navigate(
|
|||||||
.new_page(url)
|
.new_page(url)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
||||||
match wait_for_nav(&page).await {
|
// Close the tab on every exit path — the previous code closed on the two
|
||||||
Ok(()) => {}
|
// nav-error branches but leaked it on a content() read error.
|
||||||
Err(NavError::Timeout(_)) => {
|
let closer = page.clone();
|
||||||
page.close().await.ok();
|
close_after(
|
||||||
return Err(PageError::transient("nav timeout"));
|
async move {
|
||||||
}
|
closer.close().await.ok();
|
||||||
Err(NavError::Cdp(e)) => {
|
},
|
||||||
page.close().await.ok();
|
async {
|
||||||
return Err(PageError::Other(anyhow::Error::from(e)));
|
match wait_for_nav(&page).await {
|
||||||
}
|
Ok(()) => {}
|
||||||
}
|
Err(NavError::Timeout(_)) => {
|
||||||
// Best-effort wait for the page-type marker. We deliberately
|
return Err(PageError::transient("nav timeout"));
|
||||||
// discard a timeout here — see fn-level doc.
|
}
|
||||||
let _ = wait_for_selector(&page, marker, SELECTOR_TIMEOUT).await;
|
Err(NavError::Cdp(e)) => {
|
||||||
let html = page
|
return Err(PageError::Other(anyhow::Error::from(e)));
|
||||||
.content()
|
}
|
||||||
.await
|
}
|
||||||
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
// Best-effort wait for the page-type marker. We deliberately
|
||||||
page.close().await.ok();
|
// discard a timeout here — see fn-level doc.
|
||||||
classify_navigate_html(html)
|
let _ = wait_for_selector(&page, marker, SELECTOR_TIMEOUT).await;
|
||||||
|
let html = page
|
||||||
|
.content()
|
||||||
|
.await
|
||||||
|
.map_err(|e| PageError::Other(anyhow::Error::from(e)))?;
|
||||||
|
classify_navigate_html(html)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Classify a fetched body. The broken-page template is universal across
|
/// Classify a fetched body. The broken-page template is universal across
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "mangalord-frontend",
|
"name": "mangalord-frontend",
|
||||||
"version": "0.124.4",
|
"version": "0.124.5",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
Reference in New Issue
Block a user