Compare commits
2 Commits
e3cff9d874
...
4e20350645
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4e20350645 | ||
|
|
713ca139c4 |
@@ -140,7 +140,8 @@ async fn spawn_crawler_daemon(
|
||||
// authenticated without operator action.
|
||||
let mut launch_opts = cfg.browser.clone();
|
||||
if let Some(proxy) = &cfg.proxy {
|
||||
launch_opts.extra_args.push(format!("--proxy-server={proxy}"));
|
||||
let chromium_proxy = crate::crawler::url_utils::chromium_proxy_arg(proxy);
|
||||
launch_opts.extra_args.push(format!("--proxy-server={chromium_proxy}"));
|
||||
}
|
||||
let on_launch = match (&cfg.phpsessid, &cfg.cookie_domain, &cfg.start_url) {
|
||||
(Some(sid), Some(domain), Some(start_url)) => {
|
||||
|
||||
@@ -127,7 +127,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let mut options = LaunchOptions::from_env();
|
||||
if let Some(proxy) = &proxy_url {
|
||||
options.extra_args.push(format!("--proxy-server={proxy}"));
|
||||
let chromium_proxy = mangalord::crawler::url_utils::chromium_proxy_arg(proxy);
|
||||
options.extra_args.push(format!("--proxy-server={chromium_proxy}"));
|
||||
}
|
||||
let keep_open = match (keep_browser_open, options.mode) {
|
||||
(true, BrowserMode::Headed) => true,
|
||||
|
||||
@@ -91,6 +91,26 @@ pub fn registrable_domain(url: &str) -> Option<String> {
|
||||
Some(format!(".{}", registrable.join(".")))
|
||||
}
|
||||
|
||||
/// Normalise a SOCKS proxy URL for Chromium's `--proxy-server=` flag.
|
||||
///
|
||||
/// reqwest accepts both `socks5://` (resolve locally) and
|
||||
/// `socks5h://` (resolve via the SOCKS server — important when the
|
||||
/// proxy is TOR and we don't want the host's resolver to see the
|
||||
/// target hostname). Chromium does **not** know the `socks5h` scheme
|
||||
/// and refuses navigations with `ERR_NO_SUPPORTED_PROXIES`. It
|
||||
/// already sends destination hostnames over SOCKS5 by default
|
||||
/// regardless, so stripping the `h` is a pure scheme rename — the
|
||||
/// remote-DNS behaviour is preserved.
|
||||
///
|
||||
/// Non-SOCKS schemes pass through unchanged.
|
||||
pub fn chromium_proxy_arg(proxy: &str) -> String {
|
||||
if let Some(rest) = proxy.strip_prefix("socks5h://") {
|
||||
format!("socks5://{rest}")
|
||||
} else {
|
||||
proxy.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -191,4 +211,34 @@ mod tests {
|
||||
Some("[2001:db8::1]")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chromium_proxy_arg_strips_socks5h_to_socks5() {
|
||||
// Regression: passing socks5h:// to Chromium yields
|
||||
// ERR_NO_SUPPORTED_PROXIES at navigation time.
|
||||
assert_eq!(
|
||||
chromium_proxy_arg("socks5h://127.0.0.1:9050"),
|
||||
"socks5://127.0.0.1:9050"
|
||||
);
|
||||
assert_eq!(
|
||||
chromium_proxy_arg("socks5h://tor:9050"),
|
||||
"socks5://tor:9050"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chromium_proxy_arg_passes_socks5_unchanged() {
|
||||
assert_eq!(
|
||||
chromium_proxy_arg("socks5://127.0.0.1:9050"),
|
||||
"socks5://127.0.0.1:9050"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chromium_proxy_arg_passes_non_socks_unchanged() {
|
||||
assert_eq!(
|
||||
chromium_proxy_arg("http://proxy.example:8080"),
|
||||
"http://proxy.example:8080"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,5 +17,28 @@ services:
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
# Optional: TOR daemon for crawler dev. Ports bind to 127.0.0.1 only
|
||||
# — never the LAN — so a native `cargo run` on the host can reach
|
||||
# 127.0.0.1:9050 / 9051. Mirrors the prod tor service (see
|
||||
# docker-compose.yml), just with host-loopback ports and a default
|
||||
# password baked in for friction-free dev.
|
||||
tor:
|
||||
image: dockurr/tor:latest
|
||||
entrypoint: ["/bin/sh", "/usr/local/bin/mangalord-entrypoint.sh"]
|
||||
environment:
|
||||
PASSWORD: ${TOR_CONTROL_PASSWORD:-dev-tor-password}
|
||||
volumes:
|
||||
- ./tor/torrc:/etc/tor/torrc:ro
|
||||
- ./tor/entrypoint.sh:/usr/local/bin/mangalord-entrypoint.sh:ro
|
||||
ports:
|
||||
- "127.0.0.1:9050:9050"
|
||||
- "127.0.0.1:9051:9051"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "nc -z 127.0.0.1 9050 && nc -z 127.0.0.1 9051"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 30s
|
||||
|
||||
volumes:
|
||||
mangalord-postgres-dev:
|
||||
|
||||
Reference in New Issue
Block a user