Compare commits
2 Commits
e3cff9d874
...
4e20350645
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4e20350645 | ||
|
|
713ca139c4 |
@@ -140,7 +140,8 @@ async fn spawn_crawler_daemon(
|
|||||||
// authenticated without operator action.
|
// authenticated without operator action.
|
||||||
let mut launch_opts = cfg.browser.clone();
|
let mut launch_opts = cfg.browser.clone();
|
||||||
if let Some(proxy) = &cfg.proxy {
|
if let Some(proxy) = &cfg.proxy {
|
||||||
launch_opts.extra_args.push(format!("--proxy-server={proxy}"));
|
let chromium_proxy = crate::crawler::url_utils::chromium_proxy_arg(proxy);
|
||||||
|
launch_opts.extra_args.push(format!("--proxy-server={chromium_proxy}"));
|
||||||
}
|
}
|
||||||
let on_launch = match (&cfg.phpsessid, &cfg.cookie_domain, &cfg.start_url) {
|
let on_launch = match (&cfg.phpsessid, &cfg.cookie_domain, &cfg.start_url) {
|
||||||
(Some(sid), Some(domain), Some(start_url)) => {
|
(Some(sid), Some(domain), Some(start_url)) => {
|
||||||
|
|||||||
@@ -127,7 +127,8 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
let mut options = LaunchOptions::from_env();
|
let mut options = LaunchOptions::from_env();
|
||||||
if let Some(proxy) = &proxy_url {
|
if let Some(proxy) = &proxy_url {
|
||||||
options.extra_args.push(format!("--proxy-server={proxy}"));
|
let chromium_proxy = mangalord::crawler::url_utils::chromium_proxy_arg(proxy);
|
||||||
|
options.extra_args.push(format!("--proxy-server={chromium_proxy}"));
|
||||||
}
|
}
|
||||||
let keep_open = match (keep_browser_open, options.mode) {
|
let keep_open = match (keep_browser_open, options.mode) {
|
||||||
(true, BrowserMode::Headed) => true,
|
(true, BrowserMode::Headed) => true,
|
||||||
|
|||||||
@@ -91,6 +91,26 @@ pub fn registrable_domain(url: &str) -> Option<String> {
|
|||||||
Some(format!(".{}", registrable.join(".")))
|
Some(format!(".{}", registrable.join(".")))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Normalise a SOCKS proxy URL for Chromium's `--proxy-server=` flag.
|
||||||
|
///
|
||||||
|
/// reqwest accepts both `socks5://` (resolve locally) and
|
||||||
|
/// `socks5h://` (resolve via the SOCKS server — important when the
|
||||||
|
/// proxy is TOR and we don't want the host's resolver to see the
|
||||||
|
/// target hostname). Chromium does **not** know the `socks5h` scheme
|
||||||
|
/// and refuses navigations with `ERR_NO_SUPPORTED_PROXIES`. It
|
||||||
|
/// already sends destination hostnames over SOCKS5 by default
|
||||||
|
/// regardless, so stripping the `h` is a pure scheme rename — the
|
||||||
|
/// remote-DNS behaviour is preserved.
|
||||||
|
///
|
||||||
|
/// Non-SOCKS schemes pass through unchanged.
|
||||||
|
pub fn chromium_proxy_arg(proxy: &str) -> String {
|
||||||
|
if let Some(rest) = proxy.strip_prefix("socks5h://") {
|
||||||
|
format!("socks5://{rest}")
|
||||||
|
} else {
|
||||||
|
proxy.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -191,4 +211,34 @@ mod tests {
|
|||||||
Some("[2001:db8::1]")
|
Some("[2001:db8::1]")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chromium_proxy_arg_strips_socks5h_to_socks5() {
|
||||||
|
// Regression: passing socks5h:// to Chromium yields
|
||||||
|
// ERR_NO_SUPPORTED_PROXIES at navigation time.
|
||||||
|
assert_eq!(
|
||||||
|
chromium_proxy_arg("socks5h://127.0.0.1:9050"),
|
||||||
|
"socks5://127.0.0.1:9050"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
chromium_proxy_arg("socks5h://tor:9050"),
|
||||||
|
"socks5://tor:9050"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chromium_proxy_arg_passes_socks5_unchanged() {
|
||||||
|
assert_eq!(
|
||||||
|
chromium_proxy_arg("socks5://127.0.0.1:9050"),
|
||||||
|
"socks5://127.0.0.1:9050"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chromium_proxy_arg_passes_non_socks_unchanged() {
|
||||||
|
assert_eq!(
|
||||||
|
chromium_proxy_arg("http://proxy.example:8080"),
|
||||||
|
"http://proxy.example:8080"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,5 +17,28 @@ services:
|
|||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 10
|
retries: 10
|
||||||
|
|
||||||
|
# Optional: TOR daemon for crawler dev. Ports bind to 127.0.0.1 only
|
||||||
|
# — never the LAN — so a native `cargo run` on the host can reach
|
||||||
|
# 127.0.0.1:9050 / 9051. Mirrors the prod tor service (see
|
||||||
|
# docker-compose.yml), just with host-loopback ports and a default
|
||||||
|
# password baked in for friction-free dev.
|
||||||
|
tor:
|
||||||
|
image: dockurr/tor:latest
|
||||||
|
entrypoint: ["/bin/sh", "/usr/local/bin/mangalord-entrypoint.sh"]
|
||||||
|
environment:
|
||||||
|
PASSWORD: ${TOR_CONTROL_PASSWORD:-dev-tor-password}
|
||||||
|
volumes:
|
||||||
|
- ./tor/torrc:/etc/tor/torrc:ro
|
||||||
|
- ./tor/entrypoint.sh:/usr/local/bin/mangalord-entrypoint.sh:ro
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:9050:9050"
|
||||||
|
- "127.0.0.1:9051:9051"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "nc -z 127.0.0.1 9050 && nc -z 127.0.0.1 9051"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 20
|
||||||
|
start_period: 30s
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
mangalord-postgres-dev:
|
mangalord-postgres-dev:
|
||||||
|
|||||||
Reference in New Issue
Block a user