Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
f152b4d
fix: add support for name attributes in HTML fragment extraction
mre Sep 5, 2025
f165f37
feat: implement per-host rate limiting and statistics
mre Sep 7, 2025
c7480ff
fix: skip rate limiting tracking for file:// URLs
mre Sep 7, 2025
462ba32
feat: improve rate limiting logging and output formatting
mre Sep 7, 2025
7d25ea2
Fix lints
mre Sep 8, 2025
956e20a
Fix cookie jar sharing in per-host rate limiting
mre Sep 8, 2025
20844d2
Fix missing User-Agent header in per-host clients
mre Sep 8, 2025
9ecb5e9
Bring back global headers (e.g. for user-agent)
mre Sep 8, 2025
ded5bc1
Fix redirect handling in per-host clients
mre Sep 8, 2025
4577865
Pass missing args: max_redirects, timeout, allow_insecure
mre Sep 8, 2025
4e04271
Refactor host stats formatters to remove unused parameters and improv…
mre Sep 18, 2025
eed7576
remove confusing comment
mre Sep 18, 2025
ae104a0
Create `display_per_host_statistics` in separate file
mre Sep 18, 2025
8437068
Remove redundant check for `self.hosts`
mre Sep 22, 2025
92854c1
Import `std::collections::HashMap`
mre Sep 22, 2025
ada65ac
Use closures instead of if
mre Sep 22, 2025
d6f7836
Rename flags:
mre Sep 22, 2025
5960946
Fix `help` formatting
mre Sep 22, 2025
ea22e44
Reduce code duplication
thomas-zahner Oct 3, 2025
d33650d
Update documentation to reference hosts option
thomas-zahner Nov 21, 2025
dbada0d
clippy --fix
thomas-zahner Nov 27, 2025
3e0755c
Return HostPool instead of Client & code cleanup
thomas-zahner Nov 27, 2025
0939004
Move inner `Arc`s to the outside
thomas-zahner Nov 28, 2025
087ed56
Fix deadlock
thomas-zahner Nov 28, 2025
040971d
Update config options
thomas-zahner Nov 28, 2025
328ee22
Simplify host pool
thomas-zahner Dec 9, 2025
42dc072
Build host-specific reqwest clients again
thomas-zahner Dec 9, 2025
595e634
Remove max_concurrency and global_semaphore
thomas-zahner Dec 9, 2025
d374155
Update docs & reduce complexity
thomas-zahner Dec 10, 2025
3fdd992
Extract output functions
thomas-zahner Dec 10, 2025
7be4516
Replace Window with Vec
thomas-zahner Dec 10, 2025
420e822
Update RateLimitError
thomas-zahner Dec 11, 2025
00d0c13
Create RequestInterval
thomas-zahner Dec 11, 2025
08602e7
Remove RateLimitError
thomas-zahner Dec 12, 2025
75df7e8
Test and improve rate limit header handling
thomas-zahner Dec 15, 2025
3d0d4fa
Apply @mre's suggestions
thomas-zahner Dec 16, 2025
c748f7d
Apply suggestions from code review
thomas-zahner Dec 16, 2025
6297aa0
Fix tests
thomas-zahner Dec 17, 2025
a551fee
Minor improvements
thomas-zahner Dec 18, 2025
5704305
Remove reqwest_client from WebsiteChecker
thomas-zahner Dec 18, 2025
d5e8afe
Reference rate-limits crate as per @mre's suggestion
thomas-zahner Dec 18, 2025
0f21985
Update option names and the default interval value
thomas-zahner Dec 19, 2025
b815e61
Allow 0 to disable per-host rate limiting
thomas-zahner Dec 19, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Extract output functions
  • Loading branch information
thomas-zahner committed Dec 21, 2025
commit 3fdd99255563866aefdd3175718d778d066201c1
25 changes: 23 additions & 2 deletions lychee-bin/src/formatters/stats/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,38 @@ pub(crate) use raw::Raw;
use std::{
collections::{HashMap, HashSet},
fmt::Display,
fs,
io::{Write, stdout},
};

use crate::stats::ResponseStats;
use anyhow::Result;
use crate::{formatters::get_stats_formatter, options::Config, stats::ResponseStats};
use anyhow::{Context, Result};
use lychee_lib::InputSource;

pub(crate) trait StatsFormatter {
/// Format the stats of all responses and write them to stdout
fn format(&self, stats: ResponseStats) -> Result<Option<String>>;
}

/// If configured to do so, output response statistics to stdout or the specified output file.
pub(crate) fn output_response_statistics(stats: ResponseStats, config: &Config) -> Result<()> {
let is_empty = stats.is_empty();
let formatted_stats = get_stats_formatter(&config.format, &config.mode).format(stats)?;
Comment thread
thomas-zahner marked this conversation as resolved.
Outdated
if let Some(formatted_stats) = formatted_stats {
if let Some(output) = &config.output {
fs::write(output, formatted_stats).context("Cannot write status output to file")?;
} else {
if config.verbose.log_level() >= log::Level::Info && !is_empty {
// separate summary from the verbose list of links above with a newline
writeln!(stdout())?;
}
// we assume that the formatted stats don't have a final newline
writeln!(stdout(), "{formatted_stats}")?;
}
}
Ok(())
}

/// Convert a `ResponseStats` `HashMap` to a sorted Vec of key-value pairs
/// The returned keys and values are both sorted in natural, case-insensitive order
fn sort_stat_map<T>(stat_map: &HashMap<InputSource, HashSet<T>>) -> Vec<(&InputSource, Vec<&T>)>
Expand Down
4 changes: 2 additions & 2 deletions lychee-bin/src/host_stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ use lychee_lib::ratelimit::HostPool;

use crate::{formatters::get_host_stats_formatter, options::Config};

/// Display per-host statistics if requested
pub(crate) fn display_per_host_statistics(host_pool: &HostPool, config: &Config) -> Result<()> {
/// If configured to do so, output per-host statistics to stdout or the specified output file.
pub(crate) fn output_per_host_statistics(host_pool: &HostPool, config: &Config) -> Result<()> {
if !config.host_stats {
return Ok(());
}
Expand Down
63 changes: 23 additions & 40 deletions lychee-bin/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,13 @@
#![deny(missing_docs)]

use std::fs::{self, File};
use std::io::{self, BufRead, BufReader, ErrorKind, Write};
use std::io::{self, BufRead, BufReader, ErrorKind};
use std::path::PathBuf;

use anyhow::{Context, Error, Result, bail};
use clap::{Parser, crate_version};
use commands::{CommandParams, generate};
use formatters::{get_stats_formatter, log::init_logging};
use formatters::log::init_logging;
use http::HeaderMap;
use log::{error, info, warn};

Expand Down Expand Up @@ -93,11 +93,13 @@ mod stats;
mod time;
mod verbosity;

use crate::formatters::stats::output_response_statistics;
use crate::stats::ResponseStats;
use crate::{
cache::{Cache, StoreExt},
formatters::{duration::Duration, stats::StatsFormatter},
formatters::duration::Duration,
generate::generate,
host_stats::display_per_host_statistics,
host_stats::output_per_host_statistics,
options::{Config, LYCHEE_CACHE_FILE, LYCHEE_IGNORE_FILE, LycheeOptions},
};

Expand Down Expand Up @@ -381,7 +383,6 @@ async fn run(opts: &LycheeOptions) -> Result<i32> {
})?;

let client = client::create(&opts.config, cookie_jar.as_deref())?;

let params = CommandParams {
client,
cache,
Expand All @@ -393,41 +394,9 @@ async fn run(opts: &LycheeOptions) -> Result<i32> {
commands::dump(params).await?
} else {
let (stats, cache, exit_code, host_pool) = commands::check(params).await?;

let github_issues = stats
.error_map
.values()
.flatten()
.any(|body| body.uri.domain() == Some("github.com"));

let stats_formatter: Box<dyn StatsFormatter> =
get_stats_formatter(&opts.config.format, &opts.config.mode);

let is_empty = stats.is_empty();
let formatted_stats = stats_formatter.format(stats)?;

if let Some(formatted_stats) = formatted_stats {
if let Some(output) = &opts.config.output {
fs::write(output, formatted_stats).context("Cannot write status output to file")?;
} else {
if opts.config.verbose.log_level() >= log::Level::Info && !is_empty {
// separate summary from the verbose list of links above
// with a newline
writeln!(io::stdout())?;
}
// we assume that the formatted stats don't have a final newline
writeln!(io::stdout(), "{formatted_stats}")?;
}
}

// Display per-host statistics if requested
display_per_host_statistics(host_pool.as_ref(), &opts.config)?;

if github_issues && opts.config.github_token.is_none() {
warn!(
"There were issues with GitHub URLs. You could try setting a GitHub token and running lychee again.",
);
}
github_warning(&stats, &opts.config);
output_response_statistics(stats, &opts.config)?;
output_per_host_statistics(&host_pool, &opts.config)?;

if opts.config.cache {
cache.store(LYCHEE_CACHE_FILE)?;
Expand All @@ -443,3 +412,17 @@ async fn run(opts: &LycheeOptions) -> Result<i32> {

Ok(exit_code as i32)
}

/// Display user-friendly message if there were any issues with GitHub URLs
fn github_warning(stats: &ResponseStats, config: &Config) {
let github_errors = stats
.error_map
.values()
.flatten()
.any(|body| body.uri.domain() == Some("github.com"));
if github_errors && config.github_token.is_none() {
warn!(
"There were issues with GitHub URLs. You could try setting a GitHub token and running lychee again.",
);
}
}