feat: crawler scaffold with chromium launcher (0.22.0)
- crawler module (browser, source trait, jobs, diff) + binary - chromiumoxide launcher with fetcher feature (auto-downloads Chromium on first run, caches under ~/.cache/mangalord/chromium) - LaunchOptions struct with extra_args, parseable from CRAWLER_BROWSER_MODE and CRAWLER_BROWSER_ARGS - migration 0012 introduces sources, manga_sources, chapter_sources, crawler_jobs - integration tests for headed + headless launch, ipify load+parse, and extra-args propagation (all #[ignore], opt-in)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mangalord"
|
||||
version = "0.21.3"
|
||||
version = "0.22.0"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
@@ -10,6 +10,10 @@ path = "src/lib.rs"
|
||||
name = "mangalord"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "crawler"
|
||||
path = "src/bin/crawler.rs"
|
||||
|
||||
[dependencies]
|
||||
axum = { version = "0.7", features = ["macros", "multipart"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
@@ -36,7 +40,10 @@ time = "0.3"
|
||||
infer = "0.16"
|
||||
tokio-util = { version = "0.7", features = ["io"] }
|
||||
futures-core = "0.3"
|
||||
futures-util = "0.3"
|
||||
bytes = "1"
|
||||
chromiumoxide = { version = "0.7", features = ["tokio-runtime", "_fetcher-rusttls-tokio"], default-features = false }
|
||||
scraper = "0.20"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
Reference in New Issue
Block a user