diff --git a/Cargo.lock b/Cargo.lock index 4c732bc..5fcd03d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -318,6 +318,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -341,6 +350,7 @@ dependencies = [ "serde", "sqlx", "tempfile", + "time", "tokio", "tracing", "tracing-subscriber", @@ -944,6 +954,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -1110,6 +1126,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1876,6 +1898,37 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.8.0" diff --git a/Cargo.toml b/Cargo.toml index 42bdb34..4a25339 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ sqlx = { version = "0.8.2", features = [ "sqlite", ] } tempfile = "3.12.0" +time = { version = "0.3.36", features = ["formatting", "macros", "parsing"] } tokio = { version = "1.40.0", features = ["full"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } diff --git a/src/build.rs b/src/build.rs index 159bd7f..db0fb4c 100644 --- a/src/build.rs +++ b/src/build.rs @@ -15,7 +15,7 @@ use tracing::{debug, info}; use crate::{ db::{BuildMode, Db, FullBuildInfo, Status}, - nightlies::Nightlies, + nightlies::{Nightlies, NightlyCache}, }; pub struct Toolchain(String); @@ -36,8 +36,9 @@ impl Display for Toolchain { } pub async fn background_builder(db: Db) -> Result<()> { + let mut nightly_cache = NightlyCache::default(); loop { - let nightlies = Nightlies::fetch().await.wrap_err("fetching nightlies")?; + let nightlies = Nightlies::fetch(&mut nightly_cache).await.wrap_err("fetching nightlies")?; let already_finished = db .finished_nightlies() .await diff --git a/src/nightlies.rs b/src/nightlies.rs index ee6c7eb..cd26b03 100644 --- a/src/nightlies.rs +++ b/src/nightlies.rs @@ -1,21 +1,29 @@ use std::collections::HashSet; use std::hash::RandomState; -use color_eyre::eyre::Context; +use color_eyre::eyre::{Context, OptionExt}; use color_eyre::Result; +use reqwest::StatusCode; +use time::Duration; use tracing::debug; use crate::db::{BuildMode, FinishedNightly}; const EARLIEST_CUTOFF_DATE: &str = "2023-01-01"; +#[derive(Default)] +pub struct NightlyCache { + /// Nightlies that exist. + exists: HashSet, +} + /// All nightlies that exist. pub struct Nightlies { all: Vec, } impl Nightlies { - pub async fn fetch() -> Result { + pub async fn fetch(cache: &mut NightlyCache) -> Result { let manifests = reqwest::get("https://static.rust-lang.org/manifests.txt") .await .wrap_err("fetching https://static.rust-lang.org/manifests.txt")? @@ -26,8 +34,27 @@ impl Nightlies { .into_iter() .filter(|date| date.as_str() > EARLIEST_CUTOFF_DATE) .collect::>(); - all.sort_by(|a, b| b.cmp(a)); - debug!("Loaded {} nightlies from the manifest", all.len()); + + all.sort(); + + // The manifests is only updated weekly, which means new nightlies won't be contained. + // We probe for their existence. + let latest = all + .last() + .ok_or_eyre("did not find any nightlies in manifets.txt")?; + + for nightly in guess_more_recent_nightlies(&latest)? { + if nightly_exists(&nightly, cache) + .await + .wrap_err_with(|| format!("checking whether {nightly} exists"))? + { + all.push(nightly); + } + } + + all.reverse(); + + debug!("Loaded {} nightlies from the manifest and manual additions", all.len()); Ok(Self { all }) } @@ -62,6 +89,31 @@ fn nightlies_from_manifest(manifest: &str) -> Vec { .collect() } +fn guess_more_recent_nightlies(latest: &str) -> Result> { + let format = time::macros::format_description!("[year]-[month]-[day]"); + let latest = time::Date::parse(latest, format).wrap_err("latest nightly has invalid format")?; + + // manifests.txt is updated weekly, so let's try 8 just in case. + Ok((1..=8) + .filter_map(|offset| latest.checked_add(Duration::days(offset))) + .map(|date| date.format(format).unwrap()) + .collect()) +} + +async fn nightly_exists(nightly: &str, cache: &mut NightlyCache) -> Result { + if cache.exists.contains(nightly) { + return Ok(true); + } + let url = format!("https://static.rust-lang.org/dist/{nightly}/channel-rust-nightly.toml"); + let resp = reqwest::get(&url).await.wrap_err("fetching channel")?; + debug!(%nightly, %url, status = %resp.status(), "Checked whether a recent nightly exists"); + let exists = resp.status() == StatusCode::OK; + if exists { + cache.exists.insert(nightly.to_owned()); + } + Ok(exists) +} + #[cfg(test)] mod tests { #[test] @@ -74,4 +126,22 @@ static.rust-lang.org/dist/2024-08-23/channel-rust-nightly.toml"; let nightlies = super::nightlies_from_manifest(&test_manifest); assert_eq!(nightlies, vec!["2024-08-22", "2024-08-23"]); } + + #[test] + fn guess() { + let nightlies = super::guess_more_recent_nightlies("2024-08-28").unwrap(); + assert_eq!( + nightlies, + [ + "2024-08-29", + "2024-08-30", + "2024-08-31", + "2024-09-01", + "2024-09-02", + "2024-09-03", + "2024-09-04", + "2024-09-05", + ] + ); + } }