From 032e023b254468a088d9dd7c3b5a03bd814ae189 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 4 Jul 2025 17:34:41 +0200 Subject: [PATCH 1/7] Stop guessing nightlies not included in manifests.txt (#3) The file is now automatically updated after every release! --- src/build.rs | 9 +++--- src/nightlies.rs | 74 ++---------------------------------------------- 2 files changed, 7 insertions(+), 76 deletions(-) diff --git a/src/build.rs b/src/build.rs index 32fa1e1..7317e2b 100644 --- a/src/build.rs +++ b/src/build.rs @@ -15,7 +15,7 @@ use tracing::{debug, error, info}; use crate::{ db::{BuildMode, Db, FullBuildInfo, Status}, - nightlies::{Nightlies, NightlyCache}, + nightlies::Nightlies, }; pub struct Toolchain(String); @@ -36,16 +36,15 @@ impl Display for Toolchain { } pub async fn background_builder(db: Db) -> Result<()> { - let mut nightly_cache = NightlyCache::default(); loop { - if let Err(err) = background_builder_inner(&db, &mut nightly_cache).await { + if let Err(err) = background_builder_inner(&db).await { error!("error in background builder: {err}"); } } } -async fn background_builder_inner(db: &Db, nightly_cache: &mut NightlyCache) -> Result<()> { - let nightlies = Nightlies::fetch(nightly_cache) +async fn background_builder_inner(db: &Db) -> Result<()> { + let nightlies = Nightlies::fetch() .await .wrap_err("fetching nightlies")?; let already_finished = db diff --git a/src/nightlies.rs b/src/nightlies.rs index f6675dd..bbf3816 100644 --- a/src/nightlies.rs +++ b/src/nightlies.rs @@ -1,29 +1,21 @@ use std::collections::HashSet; use std::hash::RandomState; -use color_eyre::eyre::{Context, OptionExt}; +use color_eyre::eyre::Context; use color_eyre::Result; -use reqwest::StatusCode; -use time::Duration; use tracing::debug; use crate::db::{BuildMode, FinishedNightly}; const EARLIEST_CUTOFF_DATE: &str = "2022-01-01"; -#[derive(Default)] -pub struct NightlyCache { - /// Nightlies that exist. - exists: HashSet, -} - /// All nightlies that exist. pub struct Nightlies { all: Vec, } impl Nightlies { - pub async fn fetch(cache: &mut NightlyCache) -> Result { + pub async fn fetch() -> Result { let manifests = reqwest::get("https://static.rust-lang.org/manifests.txt") .await .wrap_err("fetching https://static.rust-lang.org/manifests.txt")? @@ -35,24 +27,7 @@ impl Nightlies { .filter(|date| date.as_str() > EARLIEST_CUTOFF_DATE) .collect::>(); - all.sort(); - - // The manifests is only updated weekly, which means new nightlies won't be contained. - // We probe for their existence. - let latest = all - .last() - .ok_or_eyre("did not find any nightlies in manifests.txt")?; - - for nightly in guess_more_recent_nightlies(latest)? { - if nightly_exists(&nightly, cache) - .await - .wrap_err_with(|| format!("checking whether {nightly} exists"))? - { - all.push(nightly); - } - } - - all.reverse(); + all.sort_by(|a, b| b.cmp(a)); // Reverse sort. debug!( "Loaded {} nightlies from the manifest and manual additions", @@ -92,31 +67,6 @@ fn nightlies_from_manifest(manifest: &str) -> Vec { .collect() } -fn guess_more_recent_nightlies(latest: &str) -> Result> { - let format = time::macros::format_description!("[year]-[month]-[day]"); - let latest = time::Date::parse(latest, format).wrap_err("latest nightly has invalid format")?; - - // manifests.txt is updated weekly, so let's try 8 just in case. - Ok((1..=8) - .filter_map(|offset| latest.checked_add(Duration::days(offset))) - .map(|date| date.format(format).unwrap()) - .collect()) -} - -async fn nightly_exists(nightly: &str, cache: &mut NightlyCache) -> Result { - if cache.exists.contains(nightly) { - return Ok(true); - } - let url = format!("https://static.rust-lang.org/dist/{nightly}/channel-rust-nightly.toml"); - let resp = reqwest::get(&url).await.wrap_err("fetching channel")?; - debug!(%nightly, %url, status = %resp.status(), "Checked whether a recent nightly exists"); - let exists = resp.status() == StatusCode::OK; - if exists { - cache.exists.insert(nightly.to_owned()); - } - Ok(exists) -} - #[cfg(test)] mod tests { #[test] @@ -129,22 +79,4 @@ static.rust-lang.org/dist/2024-08-23/channel-rust-nightly.toml"; let nightlies = super::nightlies_from_manifest(&test_manifest); assert_eq!(nightlies, vec!["2024-08-22", "2024-08-23"]); } - - #[test] - fn guess() { - let nightlies = super::guess_more_recent_nightlies("2024-08-28").unwrap(); - assert_eq!( - nightlies, - [ - "2024-08-29", - "2024-08-30", - "2024-08-31", - "2024-09-01", - "2024-09-02", - "2024-09-03", - "2024-09-04", - "2024-09-05", - ] - ); - } } From 3d16b9d460328c5b7d25821091f21c947155f844 Mon Sep 17 00:00:00 2001 From: Noratrieb <48135649+Noratrieb@users.noreply.github.com> Date: Fri, 4 Jul 2025 17:38:48 +0200 Subject: [PATCH 2/7] Be more clear about build kind --- templates/nightly.html | 4 ++-- templates/target.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/nightly.html b/templates/nightly.html index e4169e4..9318d77 100644 --- a/templates/nightly.html +++ b/templates/nightly.html @@ -50,8 +50,8 @@ - - + + {% for build in builds %} diff --git a/templates/target.html b/templates/target.html index af2ba20..a530a68 100644 --- a/templates/target.html +++ b/templates/target.html @@ -31,8 +31,8 @@
nightlycorestdcore
codegen		std
check
- - + + {% for build in builds %} From abb83f24b266da5e520dbcdebab46b5bdf115cd7 Mon Sep 17 00:00:00 2001 From: Noratrieb <48135649+Noratrieb@users.noreply.github.com> Date: Fri, 4 Jul 2025 17:41:47 +0200 Subject: [PATCH 3/7] don't go that far back --- src/nightlies.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nightlies.rs b/src/nightlies.rs index bbf3816..cf67977 100644 --- a/src/nightlies.rs +++ b/src/nightlies.rs @@ -7,7 +7,7 @@ use tracing::debug; use crate::db::{BuildMode, FinishedNightly}; -const EARLIEST_CUTOFF_DATE: &str = "2022-01-01"; +const EARLIEST_CUTOFF_DATE: &str = "2023-01-01"; /// All nightlies that exist. pub struct Nightlies { From 42cd9fda83c9c4995cc9105f8940f9c962a6af8f Mon Sep 17 00:00:00 2001 From: Noratrieb <48135649+Noratrieb@users.noreply.github.com> Date: Fri, 4 Jul 2025 17:55:22 +0200 Subject: [PATCH 4/7] Add filtering for failures --- .gitignore | 2 +- src/web.rs | 22 ++++++++++++++++++++++ static/build.html | 9 +++++++-- templates/nightly.html | 11 ++++++++++- templates/target.html | 17 +++++++++++++---- 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 173b3ff..54e7505 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ /target /targets /results -/db.sqlite +/db.sqlite* diff --git a/src/web.rs b/src/web.rs index 4bb4221..777c5c0 100644 --- a/src/web.rs +++ b/src/web.rs @@ -72,6 +72,7 @@ async fn web_build(State(state): State, Query(query): Query, } async fn web_target(State(state): State, Query(query): Query) -> Response { @@ -83,8 +84,11 @@ async fn web_target(State(state): State, Query(query): Query, Option)>, + showing_failures: bool, } + let filter_failures = query.failures.unwrap_or(false); + match state.db.history_for_target(&query.target).await { Ok(builds) => { let latest_core = builds @@ -122,6 +126,12 @@ async fn web_target(State(state): State, Query(query): Query>(); builds.sort_by_cached_key(|build| Reverse(build.0.clone())); @@ -130,6 +140,7 @@ async fn web_target(State(state): State, Query(query): Query, Query(query): Query, } async fn web_nightly(State(state): State, Query(query): Query) -> Response { @@ -158,8 +170,11 @@ async fn web_nightly(State(state): State, Query(query): Query, std_broken: Option, + showing_failures: bool, } + let filter_failures = query.failures.unwrap_or(false); + match state.db.history_for_nightly(&query.nightly).await { Ok(builds) => match state.db.nightly_info(&query.nightly).await { Ok(info) => { @@ -187,6 +202,12 @@ async fn web_nightly(State(state): State, Query(query): Query>(); builds.sort_by_cached_key(|build| build.0.clone()); @@ -207,6 +228,7 @@ async fn web_nightly(State(state): State, Query(query): Query

Build results for nightly-{{nightly}} target {{target}} {{mode}}

+ Home
{{status}}
@@ -20,10 +21,14 @@ {{stderr}}

- Build history for target {{target}} + Build history for target {{target}}

- Build state for nightly {{nightly}} + Build state for nightly {{nightly}}

does-it-build {{version}} diff --git a/templates/nightly.html b/templates/nightly.html index 9318d77..6176b5a 100644 --- a/templates/nightly.html +++ b/templates/nightly.html @@ -8,7 +8,7 @@

Nightly build state for {{nightly}}

- Back + Home

This contains the status of this nightly. Core is built with cargo build --release -Zbuild-std=core. This checks that @@ -47,6 +47,15 @@

{{std_failures}}

+ {% if showing_failures %} +

+ show all +

+ {% else %} +

+ filter failures +

+ {% endif %}
nightlycorestdcore
codegen		std
check
diff --git a/templates/target.html b/templates/target.html index a530a68..37aafb5 100644 --- a/templates/target.html +++ b/templates/target.html @@ -1,4 +1,4 @@ - + @@ -13,7 +13,7 @@

Target build history for {{target}}

- Back + Home
{{status}}
@@ -28,11 +28,20 @@ std builds (on targets that have it) but does not check whether codegen/linking works.

+ {% if showing_failures %} +

+ show all +

+ {% else %} +

+ filter failures +

+ {% endif %}
nightly
- - + + {% for build in builds %} From b1d1728c42d89ae24acb14bcd2c330b948f1cd7a Mon Sep 17 00:00:00 2001 From: Noratrieb <48135649+Noratrieb@users.noreply.github.com> Date: Fri, 4 Jul 2025 18:21:52 +0200 Subject: [PATCH 5/7] Support using custom rustflags for targets This is necessary for AVR and AMDGPU. We currently just hardcode some flags in the binary, which is good enough. There's a new column which keeps track of the used flags and shows them for a build. A migration makes sure that the older results are properly backfilled with the flags to make them build. --- ...0250704160556_reset-missing-target-cpu.sql | 8 +++ src/build.rs | 49 ++++++++++++++++--- src/db.rs | 6 ++- src/web.rs | 32 ++++++++---- {static => templates}/build.html | 5 ++ 5 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 migrations/20250704160556_reset-missing-target-cpu.sql rename {static => templates}/build.html (93%) diff --git a/migrations/20250704160556_reset-missing-target-cpu.sql b/migrations/20250704160556_reset-missing-target-cpu.sql new file mode 100644 index 0000000..76d96a5 --- /dev/null +++ b/migrations/20250704160556_reset-missing-target-cpu.sql @@ -0,0 +1,8 @@ +-- Add migration script here + +ALTER TABLE build_info + ADD COLUMN rustflags VARCHAR; + +DELETE FROM build_info WHERE target IN ('avr-none', 'amdgcn-amd-amdhsa') AND mode = 'core'; + +DELETE FROM finished_nightly WHERE nightly > '2025-02-11'; diff --git a/src/build.rs b/src/build.rs index 7317e2b..e2d35f2 100644 --- a/src/build.rs +++ b/src/build.rs @@ -18,6 +18,22 @@ use crate::{ nightlies::Nightlies, }; +struct CustomBuildFlags { + target: &'static str, + flags: &'static [&'static str], +} + +const CUSTOM_CORE_FLAGS: &[CustomBuildFlags] = &[ + CustomBuildFlags { + target: "avr-none", + flags: &["-Ctarget-cpu=atmega328p"], + }, + CustomBuildFlags { + target: "amdgcn-amd-amdhsa", + flags: &["-Ctarget-cpu=gfx1100"], + }, +]; + pub struct Toolchain(String); impl Toolchain { pub fn from_nightly(nightly: &str) -> Self { @@ -44,9 +60,7 @@ pub async fn background_builder(db: Db) -> Result<()> { } async fn background_builder_inner(db: &Db) -> Result<()> { - let nightlies = Nightlies::fetch() - .await - .wrap_err("fetching nightlies")?; + let nightlies = Nightlies::fetch().await.wrap_err("fetching nightlies")?; let already_finished = db .finished_nightlies() .await @@ -243,6 +257,7 @@ async fn build_single_target(db: &Db, nightly: &str, target: &str, mode: BuildMo status: result.status, stderr: result.stderr, mode, + rustflags: result.rustflags, }) .await?; @@ -252,6 +267,7 @@ async fn build_single_target(db: &Db, nightly: &str, target: &str, mode: BuildMo struct BuildResult { status: Status, stderr: String, + rustflags: Option, } /// Build a target core in a temporary directory and see whether it passes or not. @@ -261,6 +277,8 @@ async fn build_target( target: &str, mode: BuildMode, ) -> Result { + let mut rustflags = None; + let output = match mode { BuildMode::Core => { let init = Command::new("cargo") @@ -277,11 +295,22 @@ async fn build_target( std::fs::write(&librs, "#![no_std]\n") .wrap_err_with(|| format!("writing to {}", librs.display()))?; - Command::new("cargo") - .arg(format!("+{toolchain}")) + let mut cmd = Command::new("cargo"); + cmd.arg(format!("+{toolchain}")) .args(["build", "-Zbuild-std=core", "--release"]) - .args(["--target", target]) - .current_dir(tmpdir) + .args(["--target", target]); + + let extra_flags = CUSTOM_CORE_FLAGS + .iter() + .find(|flags| flags.target == target); + + if let Some(extra_flags) = extra_flags { + let flags = extra_flags.flags.join(" "); + cmd.env("RUSTFLAGS", &flags); + rustflags = Some(flags); + } + + cmd.current_dir(tmpdir) .output() .await .wrap_err("spawning cargo build")? @@ -307,5 +336,9 @@ async fn build_target( info!("Finished build"); - Ok(BuildResult { status, stderr }) + Ok(BuildResult { + status, + stderr, + rustflags, + }) } diff --git a/src/db.rs b/src/db.rs index 42c1811..73aa95f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -48,6 +48,7 @@ pub struct FullBuildInfo { pub status: Status, pub stderr: String, pub mode: BuildMode, + pub rustflags: Option, } #[derive(Debug, PartialEq, Clone, Copy, sqlx::Type, Serialize, Deserialize)] @@ -95,13 +96,14 @@ impl Db { pub async fn insert(&self, info: FullBuildInfo) -> Result<()> { sqlx::query( - "INSERT INTO build_info (nightly, target, status, stderr, mode) VALUES (?, ?, ?, ?, ?);", + "INSERT INTO build_info (nightly, target, status, stderr, mode, rustflags) VALUES (?, ?, ?, ?, ?, ?);", ) .bind(info.nightly) .bind(info.target) .bind(info.status) .bind(info.stderr) .bind(info.mode) + .bind(info.rustflags) .execute(&self.conn) .await .wrap_err("inserting build info into database")?; @@ -173,7 +175,7 @@ impl Db { mode: BuildMode, ) -> Result> { let result = sqlx::query_as::<_, FullBuildInfo>( - "SELECT nightly, target, status, stderr, mode FROM build_info + "SELECT nightly, target, status, stderr, mode, rustflags FROM build_info WHERE nightly = ? AND target = ? AND mode = ?", ) .bind(nightly) diff --git a/src/web.rs b/src/web.rs index 777c5c0..14eee79 100644 --- a/src/web.rs +++ b/src/web.rs @@ -41,6 +41,19 @@ struct BuildQuery { } async fn web_build(State(state): State, Query(query): Query) -> Response { + use askama::Template; + #[derive(askama::Template)] + #[template(path = "build.html")] + struct BuildPage { + nightly: String, + target: String, + stderr: String, + mode: BuildMode, + rustflags: Option, + version: &'static str, + status: Status, + } + match state .db .build_status_full( @@ -51,15 +64,16 @@ async fn web_build(State(state): State, Query(query): Query { - let page = include_str!("../static/build.html") - .replace("{{nightly}}", &query.nightly) - .replace("{{target}}", &query.target) - .replace("{{stderr}}", &build.stderr) - .replace("{{mode}}", &build.mode.to_string()) - .replace("{{version}}", crate::VERSION) - .replace("{{status}}", &build.status.to_string()); - - Html(page).into_response() + let page = BuildPage { + nightly: query.nightly, + target: query.target, + stderr: build.stderr, + mode: build.mode, + rustflags: build.rustflags, + version: crate::VERSION, + status: build.status, + }; + Html(page.render().unwrap()).into_response() } Ok(None) => StatusCode::NOT_FOUND.into_response(), Err(err) => { diff --git a/static/build.html b/templates/build.html similarity index 93% rename from static/build.html rename to templates/build.html index c86505d..4e28ced 100644 --- a/static/build.html +++ b/templates/build.html @@ -17,6 +17,11 @@
{{status}}
+ {% if let Some(rustflags) = rustflags %} +

+ Using rustflags: {{rustflags}} +

+ {% endif %} 
 {{stderr}}
From 9de21d41983e86fa8afe5c373c6c7b9e3deece28 Mon Sep 17 00:00:00 2001 From: Noratrieb <48135649+Noratrieb@users.noreply.github.com> Date: Fri, 4 Jul 2025 18:43:29 +0200 Subject: [PATCH 6/7] Say the build has no changes for DOES_IT_BUILD_OVERRIDE_VERSION --- build.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.rs b/build.rs index 232ad4d..ba68507 100644 --- a/build.rs +++ b/build.rs @@ -31,6 +31,10 @@ fn try_get_commit() -> color_eyre::Result { } fn has_no_changes() -> color_eyre::Result { + if std::env::var("DOES_IT_BUILD_OVERRIDE_VERSION").is_ok() { + return Ok(true); + } + Ok(std::process::Command::new("git") .args(["diff", "--no-ext-diff", "--quiet", "--exit-code"]) .output()? From 3f302c71801d6d8bca5baa57becb990aa2729b49 Mon Sep 17 00:00:00 2001 From: Noratrieb <48135649+Noratrieb@users.noreply.github.com> Date: Fri, 4 Jul 2025 18:48:16 +0200 Subject: [PATCH 7/7] Fix failure filtering for std std failures weren't getting filtered properly since I only checked core. --- src/web.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/web.rs b/src/web.rs index 14eee79..0866c46 100644 --- a/src/web.rs +++ b/src/web.rs @@ -52,7 +52,7 @@ async fn web_build(State(state): State, Query(query): Query, version: &'static str, status: Status, - } + } match state .db @@ -140,11 +140,9 @@ async fn web_target(State(state): State, Query(query): Query>(); builds.sort_by_cached_key(|build| Reverse(build.0.clone())); @@ -216,11 +214,9 @@ async fn web_nightly(State(state): State, Query(query): Query>(); builds.sort_by_cached_key(|build| build.0.clone()); @@ -324,3 +320,10 @@ impl BuildInfo { ) } } + +fn filter_build(filter_failures: bool, build: &Option) -> bool { + !filter_failures + || build + .as_ref() + .is_some_and(|build| build.status == Status::Error) +}
nightlycore
codegen		std
check		core
codegen		std
check