Creating a Robust, Reusable Link-Checker

// src/scanners/plaintext.rs

use crate::codespan::Span;
use linkify::{LinkFinder, LinkKind};

pub fn plaintext(src: &str) -> impl Iterator<Item = (&str, Span)> + '_ {
    LinkFinder::new()
        .kinds(&[LinkKind::Url])
        .links(src)
        .map(|link| {
            (
                link.as_str(),
                Span::new(link.start() as u32, link.end() as u32),
            )
        })
}

// src/scanners/plaintext.rs

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn detect_urls_in_some_text() {
        let src = "hello http://localhost/ world. this is file://some/text.";
        let should_be = vec![
            ("http://localhost/", Span::new(6, 23)),
            ("file://some/text", Span::new(39, 55)),
        ];

        let got: Vec<_> = plaintext(src).collect();

        assert_eq!(got, should_be);
    }
}

// src/scanners/markdown.rs

pub fn markdown(src: &str) -> impl Iterator<Item = (String, Span)> + '_ {
    Parser::new(src)
    .into_offset_iter()
    .filter_map(|(event, range)| match event {
        Event::Start(Tag::Link(_, dest, _))
        | Event::Start(Tag::Image(_, dest, _)) => Some((
            dest.to_string(),
            Span::new(range.start as u32, range.end as u32),
        )),
        _ => None,
    })
}

// src/scanners/markdown.rs

use crate::codespan::Span;
use pulldown_cmark::{Event, Options, Parser, Tag};

pub fn markdown(src: &str) -> impl Iterator<Item = (String, Span)> + '_ {
    markdown_with_broken_link_callback(src, &|_, _| None)
}

pub fn markdown_with_broken_link_callback<'a, F>(
    src: &'a str,
    on_broken_link: &'a F,
) -> impl Iterator<Item = (String, Span)> + 'a
where
    F: Fn(&str, &str) -> Option<(String, String)>,
{
    Parser::new_with_broken_link_callback(
        src,
        Options::ENABLE_FOOTNOTES,
        Some(on_broken_link),
    )
    .into_offset_iter()
    .filter_map(|(event, range)| match event {
        Event::Start(Tag::Link(_, dest, _))
        | Event::Start(Tag::Image(_, dest, _)) => Some((
            dest.to_string(),
            Span::new(range.start as u32, range.end as u32),
        )),
        _ => None,
    })
}

// src/validation/filesystem.rs

use std::{ffi::OsString, path::PathBuf};

pub struct Options {
    root_directory: Option<PathBuf>,
    default_file: OsString,
    links_may_traverse_the_root_directory: bool,
}

impl Options {
    pub const DEFAULT_FILE: &'static str = "index.html";

    pub fn new() -> Self {
        Options {
            root_directory: None,
            default_file: OsString::from(Options::DEFAULT_FILE),
            links_may_traverse_the_root_directory: false,
        }
    }
}

// src/validation/filesystem.rs

impl Options {
    fn join(
        &self,
        current_dir: &Path,
        second: &Path,
    ) -> Result<PathBuf, Reason> {
        if second.is_absolute() {
            // if the path is absolute (i.e. has a leading slash) then it's
            // meant to be relative to the root directory, not the current one
            match self.root_directory() {
                Some(root) => {
                    let mut buffer = root.to_path_buf();
                    // append everything except the root element
                    buffer.extend(second.iter().skip(1));
                    Ok(buffer)
                },
                // You really shouldn't provide links to absolute files on your
                // system (e.g. "/home/michael/Documents/whatever" or
                // "/etc/passwd").
                //
                // For one, it's extremely brittle and will probably only work
                // on that computer, but more importantly it's also a vector
                // for directory traversal attacks.
                //
                // Feel free to send a PR if you believe otherwise.
                None => Err(Reason::TraversesParentDirectories),
            }
        } else {
            Ok(current_dir.join(second))
        }
    }
}

// src/validation/filesystem.rs

impl Options {
    fn canonicalize(&self, path: &Path) -> Result<PathBuf, Reason> {
        let mut canonical = path.canonicalize()?;

        if canonical.is_dir() {
            canonical.push(&self.default_file);
            // we need to canonicalize again because the default file may be a
            // symlink, or not exist at all
            canonical = canonical.canonicalize()?;
        }

        Ok(canonical)
    }
}

// src/validation/filesystem.rs

impl Options {
    fn sanity_check(&self, path: &Path) -> Result<(), Reason> {
        if let Some(root) = self.root_directory() {
            if !(self.links_may_traverse_the_root_directory || path.starts_with(root))
            {
                return Err(Reason::TraversesParentDirectories);
            }
        }

        Ok(())
    }
}

// src/validation/filesystem.rs

pub fn resolve_link(
    current_directory: &Path,
    link: &Path,
    options: &Options,
) -> Result<PathBuf, Reason> {
    let joined = options.join(current_directory, link)?;

    let canonical = options.canonicalize(&joined)?;
    options.sanity_check(&canonical)?;

    // Note: canonicalizing also made sure the file exists
    Ok(canonical)
}

// src/validation/mod.rs

/// Possible reasons for a bad link.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Reason {
    #[error("Linking outside of the book directory is forbidden")]
    TraversesParentDirectories,
    #[error("An OS-level error occurred")]
    Io(#[from] std::io::Error),
    #[error("The web client encountered an error")]
    Web(#[from] reqwest::Error),
}

impl Reason {
    pub fn file_not_found(&self) -> bool {
        match self {
            Reason::Io(e) => e.kind() == std::io::ErrorKind::NotFound,
            _ => false,
        }
    }

    pub fn timed_out(&self) -> bool {
        match self {
            Reason::Web(e) => e.is_timeout(),
            _ => false,
        }
    }
}

// src/validation/mod.rs

pub trait Context {
    /// Options to use when checking a link on the filesystem.
    fn filesystem_options(&self) -> &Options;
}

// src/validation/filesystem.rs

use crate::validation::Context;

/// Check whether a [`Path`] points to a valid file on disk.
///
/// If a fragment specifier is provided, this function will scan through the
/// linked document and check that the file contains the corresponding anchor
/// (e.g. markdown heading or HTML `id`).
pub fn check_filesystem<C>(
    current_directory: &Path,
    path: &Path,
    fragment: Option<&str>,
    ctx: &C,
) -> Result<(), Reason>
where
    C: Context,
{
    log::debug!(
        "Checking \"{}\" in the context of \"{}\"",
        path.display(),
        current_directory.display()
    );

    let resolved_location = resolve_link(
        current_directory,
        path,
        ctx.filesystem_options(),
    )?;

    log::debug!(
        "\"{}\" resolved to \"{}\"",
        path.display(),
        resolved_location.display()
    );

    if let Some(fragment) = fragment {
        // TODO: detect the file type and check the fragment exists
        log::warn!(
            "Not checking that the \"{}\" section exists in \"{}\" because fragment resolution isn't implemented",
            fragment,
            resolved_location.display(),
        );
    }

    Ok(())
}

// src/validation/web.rs

use http::HeaderMap;
use reqwest::{Client, Url};

/// Send a GET request to a particular endpoint.
pub async fn get(
    client: &Client,
    url: &Url,
    extra_headers: HeaderMap,
) -> Result<(), reqwest::Error> {
    client
        .get(url.clone())
        .headers(extra_headers)
        .send()
        .await?
        .error_for_status()?;

    Ok(())
}

// src/validation/web.rs

/// Check whether a [`Url`] points to a valid resource on the internet.
pub async fn check_web<C>(url: &Url, ctx: &C) -> Result<(), Reason>
where
    C: Context,
{
    log::debug!("Checking \"{}\" on the web", url);

    if already_valid(&url, ctx) {
        log::debug!("The cache says \"{}\" is still valid", url);
        return Ok(());
    }

    let result = get(ctx.client(), &url, ctx.url_specific_headers(&url)).await;

    if let Some(fragment) = url.fragment() {
        // TODO: check the fragment
        log::warn!("Fragment checking isn't implemented, not checking if there is a \"{}\" header in \"{}\"", fragment, url);
    }

    let entry = CacheEntry::new(SystemTime::now(), result.is_ok());
    update_cache(url, ctx, entry);

    result.map_err(Reason::from)
}

// src/validation/web.rs

fn already_valid<C>(url: &Url, ctx: &C) -> bool
where
    C: Context,
{
    if let Some(cache) = ctx.cache() {
        return cache.url_is_still_valid(url, ctx.cache_timeout());
    }

    false
}

// src/validation/mod.rs

use reqwest::Url;
use std::{collections::HashMap, time::SystemTime};

pub struct Cache {
    entries: HashMap<Url, CacheEntry>,
}

pub struct CacheEntry {
    pub timestamp: SystemTime,
    pub valid: bool,
}

impl Cache {
    pub fn url_is_still_valid(&self, url: &Url, timeout: Duration) -> bool {
        if let Some(entry) = self.lookup(url) {
            if entry.valid {
                if let Ok(time_since_check_was_done) = entry.timestamp.elapsed()
                {
                    return time_since_check_was_done < timeout;
                }
            }
        }

        false
    }

    pub fn lookup(&self, url: &Url) -> Option<&CacheEntry> {
        self.entries.get(url)
    }
}

// src/validation/mod.rs

pub trait Context {
    ...

    /// An optional cache that can be used to avoid unnecessary network
    /// requests.
    ///
    /// We need to use internal mutability here because validation is done
    /// concurrently. This [`MutexGuard`] is guaranteed to be short lived (just
    /// the duration of a [`Cache::insert()`] or [`Cache::lookup()`]), so it's
    /// okay to use a [`std::sync::Mutex`] instead of [`futures::lock::Mutex`].
    fn cache(&self) -> Option<MutexGuard<Cache>> { None }

    /// How long should a cached item be considered valid for before we need to
    /// check again?
    fn cache_timeout(&self) -> Duration {
        // 24 hours should be a good default
        Duration::from_secs(24 * 60 * 60)
    }
}

// src/validation/web.rs

pub async fn check_web<C>(url: &Url, ctx: &C) -> Result<(), Reason>
where
    C: Context,
{
    ...

    let result = get(ctx.client(), &url, ctx.url_specific_headers(&url)).await;

    ...
}

// src/validation/mod.rs

pub trait Context {
    ...

    /// The HTTP client to use.
    fn client(&self) -> &Client;

    /// Get any extra headers that should be sent when checking this [`Url`].
    fn url_specific_headers(&self, _url: &Url) -> HeaderMap { HeaderMap::new() }

}

// src/validation/web.rs

pub async fn check_web<C>(url: &Url, ctx: &C) -> Result<(), Reason>
where
    C: Context,
{
    ...

    let entry = CacheEntry::new(SystemTime::now(), result.is_ok());
    update_cache(url, ctx, entry);

    ...
}

fn update_cache<C>(url: &Url, ctx: &C, entry: CacheEntry)
where
    C: Context,
{
    if let Some(mut cache) = ctx.cache() {
        cache.insert(url.clone(), entry);
    }
}

// src/validation/web.rs

pub async fn check_web<C>(url: &Url, ctx: &C) -> Result<(), Reason>
where
    C: Context,
{
    ...

    result.map_err(Reason::from)
}

// src/lib.rs

/// A link to some other resource.
#[derive(Debug, Clone, PartialEq)]
pub struct Link {
    /// The link itself.
    pub href: String,
    /// Where the [`Link`] lies in its source text.
    pub span: Span,
    /// Which document does this [`Link`] belong to?
    pub file: FileId,
}

// src/lib.rs

use std::path::PathBuf;
use reqwest::Url;

enum Category {
    /// A local file.
    FileSystem {
        path: PathBuf,
        fragment: Option<String>,
    },
    /// A URL for something on the web.
    Url(Url),
}

// src/lib.rs

#[test]
fn parse_into_categories() {
    let inputs = vec![
        (
            "https://example.com/",
            Some(Category::Url(
                Url::parse("https://example.com/").unwrap(),
            )),
        ),
        (
            "README.md",
            Some(Category::FileSystem {
                path: PathBuf::from("README.md"),
                fragment: None,
            }),
        ),
        (
            "./README.md",
            Some(Category::FileSystem {
                path: PathBuf::from("./README.md"),
                fragment: None,
            }),
        ),
        (
            "./README.md#license",
            Some(Category::FileSystem {
                path: PathBuf::from("./README.md"),
                fragment: Some(String::from("license")),
            }),
        ),
    ];

    for (src, should_be) in inputs {
        let got = Category::categorise(src);
        assert_eq!(got, should_be);
    }
}

// src/lib.rs

impl Category {
    fn categorise(src: &str) -> Option<Self> {
        if let Ok(url) = src.parse() {
            return Some(Category::Url(url));
        }

        ...
    }
}

// src/lib.rs

impl Category {
    fn categorise(src: &str) -> Option<Self> {
        ...

        let (path, fragment) = match src.find("#") {
            Some(hash) => {
                let (path, rest) = src.split_at(hash);
                (path, Some(String::from(&rest[1..])))
            },
            None => (src, None),
        };

        ...
    }
}

// src/lib.rs

impl Category {
    fn categorise(src: &str) -> Option<Self> {
        ...

        // as a sanity check we use the http crate's PathAndQuery type to make
        // sure the path is decoded correctly
        if let Ok(path_and_query) = path.parse::<PathAndQuery>() {
            return Some(Category::FileSystem {
                path: PathBuf::from(path_and_query.path()),
                fragment,
            });
        }

        ...
    }
}

// src/validation/mod.rs

/// A [`Link`] and the [`Reason`] why it is invalid.
#[derive(Debug)]
pub struct InvalidLink {
    /// The invalid link.
    pub link: Link,
    /// Why is this link invalid?
    pub reason: Reason,
}

// src/validation/mod.rs

enum Outcome {
    Valid(Link),
    Invalid(InvalidLink),
    Ignored(Link),
    UnknownCategory(Link),
}

// src/validation/mod.rs

/// Try to validate a single link, deferring to the appropriate validator based
/// on the link's [`Category`].
async fn validate_one<C>(
    link: Link,
    current_directory: &Path,
    ctx: &C,
) -> Outcome
where
    C: Context,
{
    unimplemented!()
}

// src/validation/mod.rs

pub trait Context {
    ...

    /// Should this [`Link`] be skipped?
    fn should_ignore(&self, _link: &Link) -> bool { false }
}

async fn validate_one<C>(
    link: Link,
    current_directory: &Path,
    ctx: &C,
) -> Outcome
where
    C: Context,
{
    if ctx.should_ignore(&link) {
        log::debug!("Ignoring \"{}\"", link.href);
        return Outcome::Ignored(link);
    }

    ...
}

// src/validation/mod.rs

async fn validate_one<C>(
    link: Link,
    current_directory: &Path,
    ctx: &C,
) -> Outcome
where
    C: Context,
{
    ...

    match link.category() {
        Some(Category::FileSystem { path, fragment }) => Outcome::from_result(
            link,
            check_filesystem(
                current_directory,
                &path,
                fragment.as_deref(),
                ctx,
            ),
        ),
        Some(Category::Url(url)) => {
            Outcome::from_result(link, check_web(&url, ctx).await)
        },
        None => Outcome::UnknownCategory(link),
    }
}

// src/validation/mod.rs

/// Validate several [`Link`]s relative to a particular directory.
pub fn validate<'a, L, C>(
    current_directory: &'a Path,
    links: L,
    ctx: &'a C,
) -> impl Future<Output = Outcomes> + 'a
where
    L: IntoIterator<Item = Link>,
    L::IntoIter: 'a,
    C: Context,
{
    futures::stream::iter(links)
        .map(move |link| validate_one(link, current_directory, ctx))
        .buffer_unordered(ctx.concurrency())
        .collect()
}

// src/validation/mod.rs

pub trait Context {
    ...

    /// How many items should we check at a time?
    fn concurrency(&self) -> usize { 64 }
}

/// The result of validating a batch of [`Link`]s.
#[derive(Debug, Default)]
pub struct Outcomes {
    /// Valid links.
    pub valid: Vec<Link>,
    /// Links which are broken.
    pub invalid: Vec<InvalidLink>,
    /// Items that were explicitly ignored by the [`Context`].
    pub ignored: Vec<Link>,
    /// Links which we weren't able to identify a suitable validator for.
    pub unknown_category: Vec<Link>,
}

impl Extend<Outcome> for Outcomes {
    fn extend<T: IntoIterator<Item = Outcome>>(&mut self, items: T) {
        for outcome in items {
            match outcome {
                Outcome::Valid(v) => self.valid.push(v),
                Outcome::Invalid(i) => self.invalid.push(i),
                Outcome::Ignored(i) => self.ignored.push(i),
                Outcome::UnknownCategory(u) => self.unknown_category.push(u),
            }
        }
    }
}