fix(crawler): wait for page marker instead of fixed 1s sleep (0.36.2)
A chromium snapshot taken between the wrapper-render and row-render phases let parse_chapter_list return Ok(vec![]) for a manga that actually has chapters — the soft-drop branch in sync_manga_chapters then flipped every existing chapter to dropped_at. Add wait_for_selector to crawler::nav. navigate() now takes a CSS marker matching the most-specific element the downstream parser will look for (one of LIST_PAGE_MARKER / DETAIL_PAGE_CHAPTERS_MARKER / DETAIL_PAGE_LAYOUT_MARKER). The wait is best-effort and capped by SELECTOR_TIMEOUT (10s); a legitimately empty page can still pass through because the parser's #chapter_table sentinel and the universal broken-page body check stay in force. Same pattern wired at the reader nav (a#pic_container) and probe nav (#logo), replacing the implicit assumption that the post-load JS had finished within 1 second. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -43,6 +43,48 @@ pub async fn wait_for_nav(page: &Page) -> Result<(), NavError> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll interval for [`wait_for_selector`]. 100ms is fast enough that a
|
||||
/// page rendering in 200ms isn't held back noticeably, and slow enough
|
||||
/// not to spam CDP with `find_element` calls on a page that's actually
|
||||
/// taking its time.
|
||||
const SELECTOR_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Wait until `selector` matches at least one element on `page`, or
|
||||
/// `timeout` elapses. Used after a navigation to confirm a page-type-
|
||||
/// specific marker is in the DOM before parsing — replaces the fixed
|
||||
/// post-nav sleep that previously masked partial-render races.
|
||||
///
|
||||
/// chromiumoxide 0.7.0 has no built-in `wait_for_selector`, so we poll
|
||||
/// `find_element` at [`SELECTOR_POLL_INTERVAL`] until success or budget
|
||||
/// exhaustion. A failed `find_element` is *not* an error here — it just
|
||||
/// means "not yet" — we only surface an error once the overall
|
||||
/// `timeout` is up.
|
||||
pub async fn wait_for_selector(
|
||||
page: &Page,
|
||||
selector: &str,
|
||||
timeout: Duration,
|
||||
) -> Result<(), NavError> {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
if page.find_element(selector).await.is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(NavError::Timeout(timeout));
|
||||
}
|
||||
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
let sleep_for = SELECTOR_POLL_INTERVAL.min(remaining);
|
||||
tokio::time::sleep(sleep_for).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-page-type budget for [`wait_for_selector`]. Shorter than
|
||||
/// [`NAV_TIMEOUT`] because by the time we're waiting on a selector, the
|
||||
/// page has already responded — we're only absorbing post-load JS
|
||||
/// finishing its row injection, which on a healthy site takes well
|
||||
/// under a second.
|
||||
pub const SELECTOR_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -65,4 +107,28 @@ mod tests {
|
||||
let e = NavError::Timeout(Duration::from_secs(30));
|
||||
assert_eq!(e.to_string(), "navigation timed out after 30s");
|
||||
}
|
||||
|
||||
/// Same sanity check as [`timeout_elapses_on_a_future_that_never_resolves`],
|
||||
/// but for the [`wait_for_selector`] polling pattern: the loop must
|
||||
/// surrender on `Elapsed` rather than spinning past the deadline.
|
||||
#[tokio::test(flavor = "current_thread", start_paused = true)]
|
||||
async fn selector_polling_pattern_surrenders_at_deadline() {
|
||||
let timeout = Duration::from_millis(300);
|
||||
let start = tokio::time::Instant::now();
|
||||
let deadline = start + timeout;
|
||||
// Simulate find_element forever returning "not found".
|
||||
let mut polls = 0u32;
|
||||
let result: Result<(), NavError> = loop {
|
||||
polls += 1;
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
break Err(NavError::Timeout(timeout));
|
||||
}
|
||||
tokio::time::sleep(SELECTOR_POLL_INTERVAL).await;
|
||||
};
|
||||
assert!(matches!(result, Err(NavError::Timeout(_))));
|
||||
// 300ms / 100ms poll interval ≈ 3 iterations plus the final check
|
||||
// that breaks out. Allow some slack since the first poll happens
|
||||
// before any sleep.
|
||||
assert!(polls >= 3, "expected at least 3 poll iterations, got {polls}");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user