38 lines
1.2 KiB
Rust
38 lines
1.2 KiB
Rust
use reqwest::header::{HeaderMap, USER_AGENT};
|
|
use scraper::{Html, Selector};
|
|
|
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
let args: Vec<String> = std::env::args().collect();
|
|
if args.len() < 2 {
|
|
eprintln!("Usage: {} <URL> [g]", args[0]);
|
|
return Ok(());
|
|
}
|
|
|
|
let url = &args[1];
|
|
let use_google_bot = args.get(2).map(|s| s == "g").unwrap_or(false);
|
|
|
|
let mut headers = HeaderMap::new();
|
|
if use_google_bot {
|
|
headers.insert(USER_AGENT, "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)".parse()?);
|
|
}
|
|
|
|
let client = reqwest::blocking::Client::builder().default_headers(headers).build()?;
|
|
let res = client.get(url).send()?;
|
|
|
|
let body = res.text()?;
|
|
|
|
let document = Html::parse_document(&body);
|
|
let title_selector = Selector::parse("title").unwrap();
|
|
let title = document.select(&title_selector).next().map(|title| title.text().collect::<Vec<_>>().join(""));
|
|
|
|
match title {
|
|
Some(title) => {
|
|
println!("[{}]({})", title.trim(), url);
|
|
}
|
|
None => {
|
|
eprintln!("Title tag not found in the response from {}", url);
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
} |