Add support for more recent nightlies than the manifest supports

Cargo.lock

@@ -318,6 +318,15 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "deranged"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
+dependencies = [
+ "powerfmt",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -341,6 +350,7 @@ dependencies = [
  "serde",
  "sqlx",
  "tempfile",
+ "time",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -944,6 +954,12 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "num-conv"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+
 [[package]]
 name = "num-integer"
 version = "0.1.46"
@@ -1110,6 +1126,12 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.20"
@@ -1876,6 +1898,37 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "time"
+version = "0.3.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
+dependencies = [
+ "deranged",
+ "itoa",
+ "num-conv",
+ "powerfmt",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
+
+[[package]]
+name = "time-macros"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
+dependencies = [
+ "num-conv",
+ "time-core",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.8.0"

Cargo.toml

@@ -18,6 +18,7 @@ sqlx = { version = "0.8.2", features = [
     "sqlite",
 ] }
 tempfile = "3.12.0"
+time = { version = "0.3.36", features = ["formatting", "macros", "parsing"] }
 tokio = { version = "1.40.0", features = ["full"] }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }

src/build.rs

@@ -15,7 +15,7 @@ use tracing::{debug, info};
 
 use crate::{
     db::{BuildMode, Db, FullBuildInfo, Status},
-    nightlies::Nightlies,
+    nightlies::{Nightlies, NightlyCache},
 };
 
 pub struct Toolchain(String);
@@ -36,8 +36,9 @@ impl Display for Toolchain {
 }
 
 pub async fn background_builder(db: Db) -> Result<()> {
+    let mut nightly_cache = NightlyCache::default();
     loop {
-        let nightlies = Nightlies::fetch().await.wrap_err("fetching nightlies")?;
+        let nightlies = Nightlies::fetch(&mut nightly_cache).await.wrap_err("fetching nightlies")?;
         let already_finished = db
             .finished_nightlies()
             .await

src/nightlies.rs

@@ -1,21 +1,29 @@
 use std::collections::HashSet;
 use std::hash::RandomState;
 
-use color_eyre::eyre::Context;
+use color_eyre::eyre::{Context, OptionExt};
 use color_eyre::Result;
+use reqwest::StatusCode;
+use time::Duration;
 use tracing::debug;
 
 use crate::db::{BuildMode, FinishedNightly};
 
 const EARLIEST_CUTOFF_DATE: &str = "2023-01-01";
 
+#[derive(Default)]
+pub struct NightlyCache {
+    /// Nightlies that exist.
+    exists: HashSet<String>,
+}
+
 /// All nightlies that exist.
 pub struct Nightlies {
     all: Vec<String>,
 }
 
 impl Nightlies {
-    pub async fn fetch() -> Result<Nightlies> {
+    pub async fn fetch(cache: &mut NightlyCache) -> Result<Nightlies> {
         let manifests = reqwest::get("https://static.rust-lang.org/manifests.txt")
             .await
             .wrap_err("fetching https://static.rust-lang.org/manifests.txt")?
@@ -26,8 +34,27 @@ impl Nightlies {
             .into_iter()
             .filter(|date| date.as_str() > EARLIEST_CUTOFF_DATE)
             .collect::<Vec<_>>();
-        all.sort_by(|a, b| b.cmp(a));
+
-        debug!("Loaded {} nightlies from the manifest", all.len());
+        all.sort();
+
+        // The manifests is only updated weekly, which means new nightlies won't be contained.
+        // We probe for their existence.
+        let latest = all
+            .last()
+            .ok_or_eyre("did not find any nightlies in manifets.txt")?;
+
+        for nightly in guess_more_recent_nightlies(&latest)? {
+            if nightly_exists(&nightly, cache)
+                .await
+                .wrap_err_with(|| format!("checking whether {nightly} exists"))?
+            {
+                all.push(nightly);
+            }
+        }
+
+        all.reverse();
+
+        debug!("Loaded {} nightlies from the manifest and manual additions", all.len());
         Ok(Self { all })
     }
 
@@ -62,6 +89,31 @@ fn nightlies_from_manifest(manifest: &str) -> Vec<String> {
         .collect()
 }
 
+fn guess_more_recent_nightlies(latest: &str) -> Result<Vec<String>> {
+    let format = time::macros::format_description!("[year]-[month]-[day]");
+    let latest = time::Date::parse(latest, format).wrap_err("latest nightly has invalid format")?;
+
+    // manifests.txt is updated weekly, so let's try 8 just in case.
+    Ok((1..=8)
+        .filter_map(|offset| latest.checked_add(Duration::days(offset)))
+        .map(|date| date.format(format).unwrap())
+        .collect())
+}
+
+async fn nightly_exists(nightly: &str, cache: &mut NightlyCache) -> Result<bool> {
+    if cache.exists.contains(nightly) {
+        return Ok(true);
+    }
+    let url = format!("https://static.rust-lang.org/dist/{nightly}/channel-rust-nightly.toml");
+    let resp = reqwest::get(&url).await.wrap_err("fetching channel")?;
+    debug!(%nightly, %url, status = %resp.status(), "Checked whether a recent nightly exists");
+    let exists = resp.status() == StatusCode::OK;
+    if exists {
+        cache.exists.insert(nightly.to_owned());
+    }
+    Ok(exists)
+}
+
 #[cfg(test)]
 mod tests {
     #[test]
@@ -74,4 +126,22 @@ static.rust-lang.org/dist/2024-08-23/channel-rust-nightly.toml";
         let nightlies = super::nightlies_from_manifest(&test_manifest);
         assert_eq!(nightlies, vec!["2024-08-22", "2024-08-23"]);
     }
+
+    #[test]
+    fn guess() {
+        let nightlies = super::guess_more_recent_nightlies("2024-08-28").unwrap();
+        assert_eq!(
+            nightlies,
+            [
+                "2024-08-29",
+                "2024-08-30",
+                "2024-08-31",
+                "2024-09-01",
+                "2024-09-02",
+                "2024-09-03",
+                "2024-09-04",
+                "2024-09-05",
+            ]
+        );
+    }
 }