Fix failure filtering for std

std failures weren't getting filtered properly since I only checked
core.

.gitignore

@@ -1,4 +1,4 @@
 /target
 /targets
 /results
-/db.sqlite
+/db.sqlite*

build.rs

@@ -31,6 +31,10 @@ fn try_get_commit() -> color_eyre::Result<String> {
 }
 
 fn has_no_changes() -> color_eyre::Result<bool> {
+    if std::env::var("DOES_IT_BUILD_OVERRIDE_VERSION").is_ok() {
+        return Ok(true);
+    }
+
     Ok(std::process::Command::new("git")
         .args(["diff", "--no-ext-diff", "--quiet", "--exit-code"])
         .output()?

migrations/20250704160556_reset-missing-target-cpu.sql

@@ -0,0 +1,8 @@
+-- Add migration script here
+
+ALTER TABLE build_info
+    ADD COLUMN rustflags VARCHAR;
+
+DELETE FROM build_info WHERE target IN ('avr-none', 'amdgcn-amd-amdhsa') AND mode = 'core';
+
+DELETE FROM finished_nightly WHERE nightly > '2025-02-11';

src/build.rs

@@ -15,9 +15,25 @@ use tracing::{debug, error, info};
 
 use crate::{
     db::{BuildMode, Db, FullBuildInfo, Status},
-    nightlies::{Nightlies, NightlyCache},
+    nightlies::Nightlies,
 };
 
+struct CustomBuildFlags {
+    target: &'static str,
+    flags: &'static [&'static str],
+}
+
+const CUSTOM_CORE_FLAGS: &[CustomBuildFlags] = &[
+    CustomBuildFlags {
+        target: "avr-none",
+        flags: &["-Ctarget-cpu=atmega328p"],
+    },
+    CustomBuildFlags {
+        target: "amdgcn-amd-amdhsa",
+        flags: &["-Ctarget-cpu=gfx1100"],
+    },
+];
+
 pub struct Toolchain(String);
 impl Toolchain {
     pub fn from_nightly(nightly: &str) -> Self {
@@ -36,18 +52,15 @@ impl Display for Toolchain {
 }
 
 pub async fn background_builder(db: Db) -> Result<()> {
-    let mut nightly_cache = NightlyCache::default();
     loop {
-        if let Err(err) = background_builder_inner(&db, &mut nightly_cache).await {
+        if let Err(err) = background_builder_inner(&db).await {
             error!("error in background builder: {err}");
         }
     }
 }
 
-async fn background_builder_inner(db: &Db, nightly_cache: &mut NightlyCache) -> Result<()> {
-    let nightlies = Nightlies::fetch(nightly_cache)
-        .await
-        .wrap_err("fetching nightlies")?;
+async fn background_builder_inner(db: &Db) -> Result<()> {
+    let nightlies = Nightlies::fetch().await.wrap_err("fetching nightlies")?;
     let already_finished = db
         .finished_nightlies()
         .await
@@ -244,6 +257,7 @@ async fn build_single_target(db: &Db, nightly: &str, target: &str, mode: BuildMo
         status: result.status,
         stderr: result.stderr,
         mode,
+        rustflags: result.rustflags,
     })
     .await?;
 
@@ -253,6 +267,7 @@ async fn build_single_target(db: &Db, nightly: &str, target: &str, mode: BuildMo
 struct BuildResult {
     status: Status,
     stderr: String,
+    rustflags: Option<String>,
 }
 
 /// Build a target core in a temporary directory and see whether it passes or not.
@@ -262,6 +277,8 @@ async fn build_target(
     target: &str,
     mode: BuildMode,
 ) -> Result<BuildResult> {
+    let mut rustflags = None;
+
     let output = match mode {
         BuildMode::Core => {
             let init = Command::new("cargo")
@@ -278,11 +295,22 @@ async fn build_target(
             std::fs::write(&librs, "#![no_std]\n")
                 .wrap_err_with(|| format!("writing to {}", librs.display()))?;
 
-            Command::new("cargo")
-                .arg(format!("+{toolchain}"))
+            let mut cmd = Command::new("cargo");
+            cmd.arg(format!("+{toolchain}"))
                 .args(["build", "-Zbuild-std=core", "--release"])
-                .args(["--target", target])
-                .current_dir(tmpdir)
+                .args(["--target", target]);
+
+            let extra_flags = CUSTOM_CORE_FLAGS
+                .iter()
+                .find(|flags| flags.target == target);
+
+            if let Some(extra_flags) = extra_flags {
+                let flags = extra_flags.flags.join(" ");
+                cmd.env("RUSTFLAGS", &flags);
+                rustflags = Some(flags);
+            }
+
+            cmd.current_dir(tmpdir)
                 .output()
                 .await
                 .wrap_err("spawning cargo build")?
@@ -308,5 +336,9 @@ async fn build_target(
 
     info!("Finished build");
 
-    Ok(BuildResult { status, stderr })
+    Ok(BuildResult {
+        status,
+        stderr,
+        rustflags,
+    })
 }

src/db.rs

@@ -48,6 +48,7 @@ pub struct FullBuildInfo {
     pub status: Status,
     pub stderr: String,
     pub mode: BuildMode,
+    pub rustflags: Option<String>,
 }
 
 #[derive(Debug, PartialEq, Clone, Copy, sqlx::Type, Serialize, Deserialize)]
@@ -95,13 +96,14 @@ impl Db {
 
     pub async fn insert(&self, info: FullBuildInfo) -> Result<()> {
         sqlx::query(
-            "INSERT INTO build_info (nightly, target, status, stderr, mode) VALUES (?, ?, ?, ?, ?);",
+            "INSERT INTO build_info (nightly, target, status, stderr, mode, rustflags) VALUES (?, ?, ?, ?, ?, ?);",
         )
         .bind(info.nightly)
         .bind(info.target)
         .bind(info.status)
         .bind(info.stderr)
         .bind(info.mode)
+        .bind(info.rustflags)
         .execute(&self.conn)
         .await
         .wrap_err("inserting build info into database")?;
@@ -173,7 +175,7 @@ impl Db {
         mode: BuildMode,
     ) -> Result<Option<FullBuildInfo>> {
         let result = sqlx::query_as::<_, FullBuildInfo>(
-            "SELECT nightly, target, status, stderr, mode FROM build_info
+            "SELECT nightly, target, status, stderr, mode, rustflags FROM build_info
             WHERE nightly = ? AND target = ? AND mode = ?",
         )
         .bind(nightly)

src/nightlies.rs

@@ -1,21 +1,13 @@
 use std::collections::HashSet;
 use std::hash::RandomState;
 
-use color_eyre::eyre::{Context, OptionExt};
+use color_eyre::eyre::Context;
 use color_eyre::Result;
-use reqwest::StatusCode;
-use time::Duration;
 use tracing::debug;
 
 use crate::db::{BuildMode, FinishedNightly};
 
-const EARLIEST_CUTOFF_DATE: &str = "2022-01-01";
-
-#[derive(Default)]
-pub struct NightlyCache {
-    /// Nightlies that exist.
-    exists: HashSet<String>,
-}
+const EARLIEST_CUTOFF_DATE: &str = "2023-01-01";
 
 /// All nightlies that exist.
 pub struct Nightlies {
@@ -23,7 +15,7 @@ pub struct Nightlies {
 }
 
 impl Nightlies {
-    pub async fn fetch(cache: &mut NightlyCache) -> Result<Nightlies> {
+    pub async fn fetch() -> Result<Nightlies> {
         let manifests = reqwest::get("https://static.rust-lang.org/manifests.txt")
             .await
             .wrap_err("fetching https://static.rust-lang.org/manifests.txt")?
@@ -35,24 +27,7 @@ impl Nightlies {
             .filter(|date| date.as_str() > EARLIEST_CUTOFF_DATE)
             .collect::<Vec<_>>();
 
-        all.sort();
-
-        // The manifests is only updated weekly, which means new nightlies won't be contained.
-        // We probe for their existence.
-        let latest = all
-            .last()
-            .ok_or_eyre("did not find any nightlies in manifests.txt")?;
-
-        for nightly in guess_more_recent_nightlies(latest)? {
-            if nightly_exists(&nightly, cache)
-                .await
-                .wrap_err_with(|| format!("checking whether {nightly} exists"))?
-            {
-                all.push(nightly);
-            }
-        }
-
-        all.reverse();
+        all.sort_by(|a, b| b.cmp(a)); // Reverse sort.
 
         debug!(
             "Loaded {} nightlies from the manifest and manual additions",
@@ -92,31 +67,6 @@ fn nightlies_from_manifest(manifest: &str) -> Vec<String> {
         .collect()
 }
 
-fn guess_more_recent_nightlies(latest: &str) -> Result<Vec<String>> {
-    let format = time::macros::format_description!("[year]-[month]-[day]");
-    let latest = time::Date::parse(latest, format).wrap_err("latest nightly has invalid format")?;
-
-    // manifests.txt is updated weekly, so let's try 8 just in case.
-    Ok((1..=8)
-        .filter_map(|offset| latest.checked_add(Duration::days(offset)))
-        .map(|date| date.format(format).unwrap())
-        .collect())
-}
-
-async fn nightly_exists(nightly: &str, cache: &mut NightlyCache) -> Result<bool> {
-    if cache.exists.contains(nightly) {
-        return Ok(true);
-    }
-    let url = format!("https://static.rust-lang.org/dist/{nightly}/channel-rust-nightly.toml");
-    let resp = reqwest::get(&url).await.wrap_err("fetching channel")?;
-    debug!(%nightly, %url, status = %resp.status(), "Checked whether a recent nightly exists");
-    let exists = resp.status() == StatusCode::OK;
-    if exists {
-        cache.exists.insert(nightly.to_owned());
-    }
-    Ok(exists)
-}
-
 #[cfg(test)]
 mod tests {
     #[test]
@@ -129,22 +79,4 @@ static.rust-lang.org/dist/2024-08-23/channel-rust-nightly.toml";
         let nightlies = super::nightlies_from_manifest(&test_manifest);
         assert_eq!(nightlies, vec!["2024-08-22", "2024-08-23"]);
     }
-
-    #[test]
-    fn guess() {
-        let nightlies = super::guess_more_recent_nightlies("2024-08-28").unwrap();
-        assert_eq!(
-            nightlies,
-            [
-                "2024-08-29",
-                "2024-08-30",
-                "2024-08-31",
-                "2024-09-01",
-                "2024-09-02",
-                "2024-09-03",
-                "2024-09-04",
-                "2024-09-05",
-            ]
-        );
-    }
 }

src/web.rs

@@ -41,6 +41,19 @@ struct BuildQuery {
 }
 
 async fn web_build(State(state): State<AppState>, Query(query): Query<BuildQuery>) -> Response {
+    use askama::Template;
+    #[derive(askama::Template)]
+    #[template(path = "build.html")]
+    struct BuildPage {
+        nightly: String,
+        target: String,
+        stderr: String,
+        mode: BuildMode,
+        rustflags: Option<String>,
+        version: &'static str,
+        status: Status,
+    }
+
     match state
         .db
         .build_status_full(
@@ -51,15 +64,16 @@ async fn web_build(State(state): State<AppState>, Query(query): Query<BuildQuery
         .await
     {
         Ok(Some(build)) => {
-            let page = include_str!("../static/build.html")
-                .replace("{{nightly}}", &query.nightly)
-                .replace("{{target}}", &query.target)
-                .replace("{{stderr}}", &build.stderr)
-                .replace("{{mode}}", &build.mode.to_string())
-                .replace("{{version}}", crate::VERSION)
-                .replace("{{status}}", &build.status.to_string());
-
-            Html(page).into_response()
+            let page = BuildPage {
+                nightly: query.nightly,
+                target: query.target,
+                stderr: build.stderr,
+                mode: build.mode,
+                rustflags: build.rustflags,
+                version: crate::VERSION,
+                status: build.status,
+            };
+            Html(page.render().unwrap()).into_response()
         }
         Ok(None) => StatusCode::NOT_FOUND.into_response(),
         Err(err) => {
@@ -72,6 +86,7 @@ async fn web_build(State(state): State<AppState>, Query(query): Query<BuildQuery
 #[derive(Deserialize)]
 struct TargetQuery {
     target: String,
+    failures: Option<bool>,
 }
 
 async fn web_target(State(state): State<AppState>, Query(query): Query<TargetQuery>) -> Response {
@@ -83,8 +98,11 @@ async fn web_target(State(state): State<AppState>, Query(query): Query<TargetQue
         status: String,
         version: &'static str,
         builds: Vec<(String, Option<BuildInfo>, Option<BuildInfo>)>,
+        showing_failures: bool,
     }
 
+    let filter_failures = query.failures.unwrap_or(false);
+
     match state.db.history_for_target(&query.target).await {
         Ok(builds) => {
             let latest_core = builds
@@ -122,6 +140,10 @@ async fn web_target(State(state): State<AppState>, Query(query): Query<TargetQue
             let mut builds = builds_grouped
                 .into_iter()
                 .map(|(k, (v1, v2))| (k, v1, v2))
+                .filter(|(_, core_build, std_build)| {
+                    filter_build(filter_failures, core_build)
+                        || filter_build(filter_failures, std_build)
+                })
                 .collect::<Vec<_>>();
             builds.sort_by_cached_key(|build| Reverse(build.0.clone()));
 
@@ -130,6 +152,7 @@ async fn web_target(State(state): State<AppState>, Query(query): Query<TargetQue
                 target: query.target,
                 version: crate::VERSION,
                 builds,
+                showing_failures: filter_failures,
             };
 
             Html(page.render().unwrap()).into_response()
@@ -144,6 +167,7 @@ async fn web_target(State(state): State<AppState>, Query(query): Query<TargetQue
 #[derive(Deserialize)]
 struct NightlyQuery {
     nightly: String,
+    failures: Option<bool>,
 }
 
 async fn web_nightly(State(state): State<AppState>, Query(query): Query<NightlyQuery>) -> Response {
@@ -158,8 +182,11 @@ async fn web_nightly(State(state): State<AppState>, Query(query): Query<NightlyQ
         std_failures: usize,
         core_broken: Option<String>,
         std_broken: Option<String>,
+        showing_failures: bool,
     }
 
+    let filter_failures = query.failures.unwrap_or(false);
+
     match state.db.history_for_nightly(&query.nightly).await {
         Ok(builds) => match state.db.nightly_info(&query.nightly).await {
             Ok(info) => {
@@ -187,6 +214,10 @@ async fn web_nightly(State(state): State<AppState>, Query(query): Query<NightlyQ
                 let mut builds = builds_grouped
                     .into_iter()
                     .map(|(k, (v1, v2))| (k, v1, v2))
+                    .filter(|(_, core_build, std_build)| {
+                        filter_build(filter_failures, core_build)
+                            || filter_build(filter_failures, std_build)
+                    })
                     .collect::<Vec<_>>();
                 builds.sort_by_cached_key(|build| build.0.clone());
 
@@ -207,6 +238,7 @@ async fn web_nightly(State(state): State<AppState>, Query(query): Query<NightlyQ
                     core_failures,
                     core_broken,
                     std_broken,
+                    showing_failures: filter_failures,
                 };
 
                 Html(page.render().unwrap()).into_response()
@@ -288,3 +320,10 @@ impl BuildInfo {
         )
     }
 }
+
+fn filter_build(filter_failures: bool, build: &Option<BuildInfo>) -> bool {
+    !filter_failures
+        || build
+            .as_ref()
+            .is_some_and(|build| build.status == Status::Error)
+}

templates/build.html

@@ -13,17 +13,27 @@
   </head>
   <body>
     <h1>Build results for nightly-{{nightly}} target {{target}} {{mode}}</h1>
+    <a href="/">Home</a>
     <div style="margin-top: 20px" class="{{status}} build-indicator-big">
       {{status}}
     </div>
+    {% if let Some(rustflags) = rustflags %}
+    <p>
+      Using rustflags: <b><code>{{rustflags}}</code></b>
+    </p>
+    {% endif %}
     <pre>
 {{stderr}}
     </pre>
     <p>
-      <a href="/target?target={{target}}">Build history for target {{target}}</a>
+      <a href="/target?target={{target}}"
+        >Build history for target {{target}}</a
+      >
     </p>
     <p>
-      <a href="/nightly?nightly={{nightly}}">Build state for nightly {{nightly}}</a>
+      <a href="/nightly?nightly={{nightly}}"
+        >Build state for nightly {{nightly}}</a
+      >
     </p>
     <footer class="footer">
       <span>does-it-build {{version}}</span>

templates/nightly.html

@@ -8,7 +8,7 @@
   </head>
   <body>
     <h1>Nightly build state for {{nightly}}</h1>
-    <a href="/">Back</a>
+    <a href="/">Home</a>
     <p>
       This contains the status of this nightly. Core is built with
       <code>cargo build --release -Zbuild-std=core</code>. This checks that
@@ -47,11 +47,20 @@
         <dd>{{std_failures}}</dd>
       </dl>
     </p>
+    {% if showing_failures %}
+    <p>
+      <a href="/nightly?nightly={{nightly}}">show all</a>
+    </p>
+    {% else %}
+    <p>
+      <a href="/nightly?nightly={{nightly}}&failures=true">filter failures</a>
+    </p>
+    {% endif %}
     <table>
       <tr>
         <th>nightly</th>
-        <th>core</th>
-        <th>std</th>
+        <th>core<br>codegen</th>
+        <th>std<br>check</th>
       </tr>
       {% for build in builds %}
       <tr>

templates/target.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
@@ -13,7 +13,7 @@
   </head>
   <body>
     <h1>Target build history for {{target}}</h1>
-    <a href="/">Back</a>
+    <a href="/">Home</a>
     <div style="margin-top: 20px" class="{{status}} build-indicator-big">
       {{status}}
     </div>
@@ -28,11 +28,20 @@
       std builds (on targets that have it) but does not check whether
       codegen/linking works.
     </p>
+    {% if showing_failures %}
+    <p>
+      <a href="/target?target={{target}}">show all</a>
+    </p>
+    {% else %}
+    <p>
+      <a href="/target?target={{target}}&failures=true">filter failures</a>
+    </p>
+    {% endif %}
     <table>
       <tr>
         <th>nightly</th>
-        <th>core</th>
-        <th>std</th>
+        <th>core<br />codegen</th>
+        <th>std<br />check</th>
       </tr>
       {% for build in builds %}
       <tr>