Extracting Links

Extract all links from a webpage HTML

reqwest scraper tokio thiserror Net

Use reqwest::get to perform a HTTP GET request and then use Html::parse_document to parse the response into a HTML document. Selector::parse compiles the CSS selector a[href], which matches every link that has an “href” attribute. Html::select yields the matching elements, and filter_map retrieves each URL with attr on the element’s ElementRef::value.

This recipe requires the macros and rt-multi-thread feature flags of tokio to be enabled in Cargo.toml.

#![allow(unused)]
fn main() {
use scraper::{Html, Selector};
use thiserror::Error;

#[derive(Error, Debug)]
pub enum LinkError {
    #[error("Reqwest error: {0}")]
    ReqError(#[from] reqwest::Error),
    #[error("IO error: {0}")]
    IoError(#[from] std::io::Error),
}

fn extract_links(html: &str) -> Vec<Box<str>> {
    let document = Html::parse_document(html);
    let selector = Selector::parse("a[href]").expect("valid CSS selector");
    document
        .select(&selector)
        .filter_map(|element| element.value().attr("href"))
        .map(Box::<str>::from)
        .collect()
}

pub async fn get_links(page: &str) -> Result<Vec<Box<str>>, LinkError> {
    let res = reqwest::get(page).await?.text().await?;
    Ok(extract_links(&res))
}
}

use web_scraping_example::links;

#[tokio::main]
async fn main() -> Result<(), links::LinkError> {
    let page_links = links::get_links("https://www.rust-lang.org/en-US/").await?;
    for link in page_links {
        println!("{}", link);
    }
    Ok(())
}

Check a webpage for broken links

reqwest scraper url tokio thiserror Net

get_links parses the page with Html::parse_document and calls get_base_url to retrieve the base URL. If the document has a base tag, get the href attr from the base[href] element, which Selector::parse and Html::select find with a CSS selector. Position::BeforePath of the original URL acts as a default.

Iterates through links in the document and creates a tokio::task::spawn task that will parse an individual link with url::ParseOptions and Url::parse). The task makes a request to the links with reqwest and verifies StatusCode. Then the tasks await completion before ending the program.

This recipe requires the macros and rt-multi-thread feature flags of tokio to be enabled in Cargo.toml.

#![allow(unused)]
fn main() {
use reqwest::StatusCode;
use scraper::{Html, Selector};
use std::collections::HashSet;
use thiserror::Error;
use url::{Position, Url};

#[derive(Error, Debug)]
pub enum BrokenError {
    #[error("Reqwest error: {0}")]
    ReqError(#[from] reqwest::Error),
    #[error("IO error: {0}")]
    IoError(#[from] std::io::Error),
    #[error("URL parse error: {0}")]
    UrlParseError(#[from] url::ParseError),
    #[error("Join error: {0}")]
    JoinError(#[from] tokio::task::JoinError),
}

pub struct CategorizedUrls {
    pub ok: Vec<String>,
    pub broken: Vec<String>,
}

enum Link {
    GoodLink(Url),
    BadLink(Url),
}

fn get_base_url(url: &Url, document: &Html) -> Result<Url, BrokenError> {
    let selector = Selector::parse("base[href]").expect("valid CSS selector");
    let base_tag_href = document
        .select(&selector)
        .filter_map(|element| element.value().attr("href"))
        .next();
    let base_url =
        base_tag_href.map_or_else(|| Url::parse(&url[..Position::BeforePath]), Url::parse)?;
    Ok(base_url)
}

fn get_links(url: &Url, body: &str) -> Result<HashSet<Url>, BrokenError> {
    let document = Html::parse_document(body);
    let base_url = get_base_url(url, &document)?;
    let base_parser = Url::options().base_url(Some(&base_url));
    let selector = Selector::parse("a[href]").expect("valid CSS selector");
    let links = document
        .select(&selector)
        .filter_map(|element| element.value().attr("href"))
        .filter_map(|link| base_parser.parse(link).ok())
        .collect();
    Ok(links)
}

async fn check_link(url: &Url) -> Result<bool, BrokenError> {
    let res = reqwest::get(url.as_ref()).await?;
    Ok(res.status() != StatusCode::NOT_FOUND)
}

pub async fn check(site: &str) -> Result<CategorizedUrls, BrokenError> {
    let url = Url::parse(site)?;
    let res = reqwest::get(url.as_ref()).await?.text().await?;
    let links = get_links(&url, &res)?;
    let mut tasks = vec![];
    let mut ok = vec![];
    let mut broken = vec![];

    for link in links {
        tasks.push(tokio::spawn(async move {
            if check_link(&link).await.unwrap_or(false) {
                Link::GoodLink(link)
            } else {
                Link::BadLink(link)
            }
        }));
    }

    for task in tasks {
        match task.await? {
            Link::GoodLink(link) => ok.push(link.to_string()),
            Link::BadLink(link) => broken.push(link.to_string()),
        }
    }

    Ok(CategorizedUrls { ok, broken })
}
}

use web_scraping_example::broken;

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let categorized = broken::check("https://www.rust-lang.org/en-US/").await?;
    println!("OK: {:?}", categorized.ok);
    println!("Broken: {:?}", categorized.broken);
    Ok(())
}

Extract all unique links from a MediaWiki markup

reqwest regex tokio Net

Pull the source of a MediaWiki page using reqwest::get and then look for all entries of internal and external links with Regex::captures_iter. Using Cow avoids excessive String allocations.

MediaWiki link syntax is described here. The calling function will retain the whole document, and links will be returned as slice references to the original document. The request sets a user agent, since Wikipedia rejects requests without one.

This recipe requires the macros and rt-multi-thread feature flags of tokio to be enabled in Cargo.toml.

#![allow(unused)]
fn main() {
use regex::Regex;
use std::borrow::Cow;
use std::collections::HashSet;
use std::sync::LazyLock;

pub fn extract_links(content: &str) -> HashSet<Cow<'_, str>> {
    static WIKI_REGEX: LazyLock<Regex> = LazyLock::new(|| {
        Regex::new(
            r"(?x)
                \[\[(?P<internal>[^\[\]|]*)[^\[\]]*\]\]    # internal links
                |
                (url=|URL\||\[)(?P<external>http.*?)[ \|}] # external links
            ",
        )
        .unwrap()
    });

    let links: HashSet<_> = WIKI_REGEX
        .captures_iter(content)
        .map(|c| match (c.name("internal"), c.name("external")) {
            (Some(val), None) => Cow::from(val.as_str()),
            (None, Some(val)) => Cow::from(val.as_str()),
            _ => unreachable!(),
        })
        .collect::<HashSet<_>>();

    links
}
}

use web::wiki;

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let content = reqwest::Client::builder()
        .user_agent("rust-cookbook")
        .build()?
        .get("https://en.wikipedia.org/w/index.php?title=Rust_(programming_language)&action=raw")
        .send()
        .await?
        .text()
        .await?;

    println!("{:#?}", wiki::extract_links(content.as_str()));

    Ok(())
}