Add support for local files #262

.github/workflows/release.yml

@@ -43,17 +43,17 @@ jobs:
       fail-fast: false
     steps:
       - name: Install musl tools
-        if: contains(matrix.target, 'musl')
+        if: ${{ contains(matrix.target, 'musl') }}
         run: sudo apt-get install -y musl-tools
 
       - name: Install arm tools
-        if: contains(matrix.target, 'arm')
+        if: ${{ contains(matrix.target, 'arm') }}
         run: |
           echo "GNU_PREFIX=arm-linux-gnueabihf-" >> $GITHUB_ENV
           sudo apt-get install -y binutils-arm-linux-gnueabihf   
       
       - name: Install aarch64 tools
-        if: contains(matrix.target, 'aarch64')
+        if: ${{ contains(matrix.target, 'aarch64') }}
         run: |
           echo "GNU_PREFIX=aarch64-linux-gnu-" >> $GITHUB_ENV
           sudo apt-get install -y binutils-aarch64-linux-gnu 

.github/workflows/rust.yml

@@ -56,7 +56,7 @@ jobs:
     - run: cargo-publish-all --dry-run
 
   publish:
-    if: startsWith(github.ref, 'refs/tags/')
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
     needs:
       - test
       - lint

Makefile

@@ -18,10 +18,14 @@ docker-run: ## Run Docker image
 docker-push: ## Push image to Docker Hub
 	docker push $(IMAGE_NAME)
 
-.PHONY: build-local
+.PHONY: build
 build: ## Build Rust code locally
 	cargo build
 
+.PHONY: install
+install: ## Install project locally
+	cargo install --path lychee-bin
+
 .PHONY: run
 run: ## Run Rust code locally
 	cargo run

README.md

@@ -161,11 +161,15 @@ lychee ~/projects/*/README.md
 
 # check links in local files (lychee supports advanced globbing and ~ expansion):
 lychee "~/projects/big_project/**/README.*"
+
 # ignore case when globbing and check result for each link:
 lychee --glob-ignore-case --verbose "~/projects/**/[r]eadme.*"
 
 # check links from epub file (requires atool: https://www.nongnu.org/atool)
 acat -F zip {file.epub} "*.xhtml" "*.html" | lychee -
+
+# check links in directory; block network requests
+lychee --offline path/to/directory
 ```
 
 ### GitHub token
@@ -202,14 +206,16 @@ FLAGS:
     -i, --insecure               Proceed for server connections considered insecure (invalid TLS)
     -n, --no-progress            Do not show progress bar.
                                  This is recommended for non-interactive shells (e.g. for continuous integration)
+        --offline                Only check local files and block network requests
         --require-https          When HTTPS is available, treat HTTP links as errors
         --skip-missing           Skip missing input files (default is to error if they don't exist)
     -V, --version                Prints version information
     -v, --verbose                Verbose program output
 
 OPTIONS:
     -a, --accept <accept>                      Comma-separated list of accepted status codes for valid links
-    -b, --base-url <base-url>                  Base URL to check relative URLs
+    -b, --base <base>                          Base URL or website root directory to check relative URLs e.g.
+                                               https://example.org or `/path/to/public`
         --basic-auth <basic-auth>              Basic authentication support. E.g. `username:password`
     -c, --config <config-file>                 Configuration file to use [default: ./lychee.toml]
         --exclude <exclude>...                 Exclude URLs from checking (supports regex)
@@ -310,7 +316,8 @@ Try one of these links to get started:
 - [good first issues](https://github.com/lycheeverse/lychee/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22)
 - [help wanted](https://github.com/lycheeverse/lychee/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22)
 
-Lychee is written in Rust. Install [rust-up](https://rustup.rs/) to get started. Begin my making sure the following commands succeed without errors.
+Lychee is written in Rust. Install [rust-up](https://rustup.rs/) to get started.
+Begin my making sure the following commands succeed without errors.
 
 ```bash
 cargo test # runs tests

examples/collect_links/collect_links.rs

@@ -14,12 +14,12 @@ async fn main() -> Result<()> {
     ];
 
     let links = Collector::new(
-        None,  // base_url
+        None,  // base
         false, // don't skip missing inputs
         10,    // max concurrency
     )
     .collect_links(
-        inputs, // base_url
+        inputs, // base url or directory
     )
     .await?;
 

fixtures/TEST.md

@@ -1,5 +1,5 @@
-This link should be ignored as it is not a fully qualified URL.
-![Logo](awesome.png)
+Check file link
+![Logo](../assets/banner.svg)
 
 ![Anchors should be ignored](#awesome)
 

fixtures/TEST_SCHEMES.txt

@@ -1,3 +1,3 @@
 slack://channel?id=123
-file://foo/bar
+file:///test_folder/test_file
 https://example.org

fixtures/offline/about/index.html

@@ -0,0 +1,21 @@
+<html>
+    <head>
+        <title>About</title>
+    </head>
+    <body>
+        <h1>About</h1>
+        <p>
+            <ul>
+                <li>
+                    <a href="https://example.org">example</a>
+                </li>
+                <li>
+                    <a href="/">home</a>
+                </li>
+                <li>
+                    <a href="/post1">Post 1</a>
+                </li>
+            </ul>
+        </p>
+    </body>
+</html>

fixtures/offline/blog/post1/index.html

@@ -0,0 +1,21 @@
+<html>
+    <head>
+        <title>Post 2</title>
+    </head>
+    <body>
+        <h1>Post 2 Title</h1>
+        <p>
+            <ul>
+                <li>
+                    <a href="/">home</a>
+                </li>
+                <li>
+                    <a href="/post1">Post 1</a>
+                </li>
+                <li>
+                    <a href="../about">Relative</a>
+                </li>
+            </ul>
+        </p>
+    </body>
+</html>

fixtures/offline/blog/post2/index.html

@@ -0,0 +1,18 @@
+<html>
+    <head>
+        <title>Post 1</title>
+    </head>
+    <body>
+        <h1>Post 1 Title</h1>
+        <p>
+            <ul>
+                <li>
+                    <a href="/">home</a>
+                </li>
+                <li>
+                    <a href="/post2">Post 2</a>
+                </li>
+            </ul>
+        </p>
+    </body>
+</html>

fixtures/offline/index.html

@@ -0,0 +1,27 @@
+<html>
+    <head>
+        <title>Index</title>
+    </head>
+    <body>
+        <h1>Index Title</h1>
+        <p>
+            <ul>
+                <li>
+                    <a href="/">home</a>
+                </li>
+                <li>
+                    <a href="/about">About</a>
+                </li>
+                <li>
+                    <a href="/about#fragment">About</a>
+                </li>
+                <li>
+                    <a href="/another page">About</a>
+                </li>
+                <li>
+                    <a href="/another%20page">About</a>
+                </li>
+            </ul>
+        </p>
+    </body>
+</html>

lychee-bin/src/main.rs

@@ -70,10 +70,7 @@ use anyhow::{anyhow, Context, Result};
 use headers::{authorization::Basic, Authorization, HeaderMap, HeaderMapExt, HeaderName};
 use http::StatusCode;
 use indicatif::{ProgressBar, ProgressStyle};
-use lychee_lib::{
-    collector::{Collector, Input},
-    ClientBuilder, ClientPool, Response,
-};
+use lychee_lib::{ClientBuilder, ClientPool, Collector, Input, Response};
 use openssl_sys as _; // required for vendored-openssl feature
 use regex::RegexSet;
 use ring as _; // required for apple silicon
@@ -178,6 +175,13 @@ async fn run(cfg: &Config, inputs: Vec<Input>) -> Result<i32> {
     let include = RegexSet::new(&cfg.include)?;
     let exclude = RegexSet::new(&cfg.exclude)?;
 
+    // Offline mode overrides the scheme
+    let schemes = if cfg.offline {
+        vec!["file".to_string()]
+    } else {
+        cfg.scheme.clone()
+    };
+
     let client = ClientBuilder::builder()
         .includes(include)
         .excludes(exclude)
@@ -193,14 +197,14 @@ async fn run(cfg: &Config, inputs: Vec<Input>) -> Result<i32> {
         .method(method)
         .timeout(timeout)
         .github_token(cfg.github_token.clone())
-        .schemes(HashSet::from_iter(cfg.scheme.clone()))
+        .schemes(HashSet::from_iter(schemes))
         .accepted(accepted)
         .require_https(cfg.require_https)
         .build()
         .client()
         .map_err(|e| anyhow!(e))?;
 
-    let links = Collector::new(cfg.base_url.clone(), cfg.skip_missing, max_concurrency)
+    let links = Collector::new(cfg.base.clone(), cfg.skip_missing, max_concurrency)
         .collect_links(&inputs)
         .await
         .map_err(|e| anyhow!(e))?;

lychee-bin/src/options.rs

@@ -1,9 +1,8 @@
-use std::{fs, io::ErrorKind, path::PathBuf, str::FromStr};
+use std::{convert::TryFrom, fs, io::ErrorKind, path::PathBuf, str::FromStr};
 
 use anyhow::{anyhow, Error, Result};
 use lazy_static::lazy_static;
-use lychee_lib::collector::Input;
-use reqwest::Url;
+use lychee_lib::{Base, Input};
 use serde::Deserialize;
 use structopt::{clap::crate_version, StructOpt};
 
@@ -76,6 +75,10 @@ macro_rules! fold_in {
     };
 }
 
+fn parse_base(src: &str) -> Result<Base, lychee_lib::ErrorKind> {
+    Base::try_from(src)
+}
+
 #[derive(Debug, StructOpt)]
 #[structopt(
     name = "lychee",
@@ -161,6 +164,11 @@ pub(crate) struct Config {
     #[serde(default)]
     pub(crate) scheme: Vec<String>,
 
+    /// Only check local files and block network requests.
+    #[structopt(long)]
+    #[serde(default)]
+    pub(crate) offline: bool,
+
     /// URLs to check (supports regex). Has preference over all excludes.
     #[structopt(long)]
     #[serde(default)]
@@ -223,10 +231,11 @@ pub(crate) struct Config {
     #[serde(default = "method")]
     pub(crate) method: String,
 
-    /// Base URL to check relative URLs
-    #[structopt(short, long, parse(try_from_str))]
+    /// Base URL or website root directory to check relative URLs
+    /// e.g. https://example.org or `/path/to/public`
+    #[structopt(short, long, parse(try_from_str = parse_base))]
     #[serde(default)]
-    pub(crate) base_url: Option<Url>,
+    pub(crate) base: Option<Base>,
 
     /// Basic authentication support. E.g. `username:password`
     #[structopt(long)]
@@ -311,7 +320,7 @@ impl Config {
             accept: None;
             timeout: TIMEOUT;
             method: METHOD;
-            base_url: None;
+            base: None;
             basic_auth: None;
             github_token: None;
             skip_missing: false;