From de7eca4ab167f3a61b36f6bbc89685be90f384db Mon Sep 17 00:00:00 2001 From: nils <48135649+Nilstrieb@users.noreply.github.com> Date: Thu, 29 Sep 2022 08:25:07 +0200 Subject: [PATCH 01/18] Update main.rs --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 90d90de..05064d2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,7 +36,7 @@ async fn main() -> color_eyre::Result<()> { ) .init(); - let (job_queue_send, job_queue_recv) = mpsc::channel(); + let (job_queue_send, job_queue_recv) = mpsc::channel(); // FIXME: make this a sync_channel because bounds are cool let sqlite_db = env::var("SQLITE_DB").unwrap_or_else(|_| "bisect.sqlite".to_string()); From f391b59f79393de09a72426565b560db6299283e Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Thu, 29 Sep 2022 18:24:32 +0200 Subject: [PATCH 02/18] run.sh --- run.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 run.sh diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..561ef51 --- /dev/null +++ b/run.sh @@ -0,0 +1,4 @@ +docker run -d --name cargo-bisect-rustc-service --net=internal --restart=always \ + "-v=/apps/cargo-bisect-rustc-service/db:/app/db" \ + "-e=SQLITE_DB=/app/db/db.sqlite" \ + docker.nilstrieb.dev/cargo-bisect-rustc-service:1.3 \ No newline at end of file From b95c9d4b44bedacadf3abc4a76833568b9867646 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Thu, 29 Sep 2022 20:14:17 +0200 Subject: [PATCH 03/18] improvements --- index.html | 32 +++++++++++++++++-- src/bisect.rs | 87 +++++++++++++++++++++++++++++---------------------- src/main.rs | 1 + 3 files changed, 79 insertions(+), 41 deletions(-) diff --git a/index.html b/index.html index 6a43da1..daa30e0 100644 --- a/index.html +++ b/index.html @@ -34,13 +34,28 @@ cols="80" placeholder="// Rust code goes here..." > -fn main() {}
- + + +

@@ -64,6 +79,7 @@ fn main() {}, conn: Connection) { Err(_) => return, }; - info!(id = %job.id, "Starting bisection job"); - - let mut bisect = Bisection { - id: job.id, - code: job.code.clone(), - status: BisectStatus::InProgress, - }; - - match db::add_bisection(&conn, &bisect).wrap_err("insert bisection") { - Ok(()) => { - let status = match bisect_job(job) { - Ok(status) => status, - Err(err) => { - error!(?err, "error processing bisection"); - BisectStatus::Error { - output: err.to_string(), - } - } - }; - - bisect.status = status; - - match db::update_bisection_status(&conn, &bisect) { - Ok(()) => {} - Err(err) => error!(?err, "error updating bisection"), - } + match process_job(job, &conn) { + Ok(()) => {} + Err(err) => { + error!(?err, "error processing bisection") } - Err(err) => error!(?err, "error inserting bisection"), } } } -#[tracing::instrument(skip(job), fields(id = %job.id))] +#[tracing::instrument(skip(job, conn), fields(id = %job.id))] +pub fn process_job(job: Job, conn: &Connection) -> Result<()> { + info!(id = %job.id, "Starting bisection job"); + + let mut bisect = Bisection { + id: job.id, + code: job.code.clone(), + status: BisectStatus::InProgress, + }; + + db::add_bisection(&conn, &bisect).wrap_err("insert bisection")?; + + let status = match bisect_job(job) { + Ok(status) => status, + Err(err) => { + error!(?err, "error processing bisection"); + BisectStatus::Error { + output: format!("Internal error"), + } + } + }; + + bisect.status = status; + + db::update_bisection_status(&conn, &bisect).wrap_err("writing bisection result")?; + + trace!(?bisect, "Finished bisection job"); + + Ok(()) +} + fn bisect_job(job: Job) -> Result { - let (output, state) = run_bisect_for_file(job.code, job.options.start, job.options.end)?; + let (output, state) = run_bisect_for_file(job.code, &job.options)?; info!(state = %state.status(), "Bisection finished"); process_result(output, state).wrap_err("process result") @@ -113,7 +120,9 @@ fn process_result(output: Output, state: JobState) -> Result { match state { JobState::Failed => { - let output = stderr.lines().rev().take(10).collect::(); + let mut output = stderr.lines().rev().take(30).collect::>(); + output.reverse(); + let output = output.join("\n"); info!(?output, "output"); Ok(BisectStatus::Error { output }) } @@ -127,11 +136,7 @@ fn process_result(output: Output, state: JobState) -> Result { } } -fn run_bisect_for_file( - input: String, - start: chrono::NaiveDate, - end: Option, -) -> Result<(Output, JobState)> { +fn run_bisect_for_file(input: String, options: &Options) -> Result<(Output, JobState)> { let temp_dir = tempdir::TempDir::new("bisect").wrap_err("creating tempdir")?; let mut cargo_new = Command::new("cargo"); cargo_new @@ -156,12 +161,18 @@ fn run_bisect_for_file( bisect.arg("--timeout").arg("30"); // don't hang bisect.current_dir(&cargo_dir); - bisect.arg("--start").arg(start.to_string()); + bisect.arg("--start").arg(options.start.to_string()); - if let Some(end) = end { + if let Some(end) = options.end { bisect.arg("--end").arg(end.to_string()); } + bisect + .arg("--regress") + .arg(options.kind.as_deref().unwrap_or("ice")); // FIXME Make this configurable + + bisect.env("RUST_LOG", "error"); // overwrite RUST_LOG + let output = bisect.output().wrap_err("spawning cargo-bisect-rustc")?; if output.status.success() { diff --git a/src/main.rs b/src/main.rs index 90d90de..9e2b075 100644 --- a/src/main.rs +++ b/src/main.rs @@ -97,6 +97,7 @@ async fn get_bisections(Extension(conn): Extension) -> impl IntoResponse { pub struct Options { start: chrono::NaiveDate, end: Option, + kind: Option, } #[derive(Debug, Serialize)] From 41396f19c7b8b6a9ff939447a63f96a9436f56ce Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Thu, 29 Sep 2022 20:19:14 +0200 Subject: [PATCH 04/18] fmt --- index.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/index.html b/index.html index daa30e0..56ff509 100644 --- a/index.html +++ b/index.html @@ -56,8 +56,10 @@ impl<T> Struct<T> { +

+
From ae9f992ef8c9a2b97da0c2a6714ab2b84d8db835 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Thu, 29 Sep 2022 21:52:56 +0200 Subject: [PATCH 05/18] time and error handling --- Cargo.lock | 1 + Cargo.toml | 4 +- index.html | 101 ++++++++++++++++++++++++++++++++++---------------- src/bisect.rs | 5 ++- src/db.rs | 19 ++++++---- 5 files changed, 88 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b91c155..4e5d009 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -748,6 +748,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" dependencies = [ "bitflags", + "chrono", "fallible-iterator", "fallible-streaming-iterator", "hashlink", diff --git a/Cargo.toml b/Cargo.toml index 71c5a81..7c9473a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,9 +7,9 @@ edition = "2021" [dependencies] axum = "0.5.16" -chrono = { version = "0.4.22", features = ["serde"] } +chrono = { version = "0.4.22", features = ["serde", "clock"] } color-eyre = "0.6.2" -rusqlite = { version = "0.28.0", features = ["bundled", "uuid"] } +rusqlite = { version = "0.28.0", features = ["bundled", "uuid", "chrono"] } serde = { version = "1.0.145", features = ["derive"] } tempdir = "0.3.7" tokio = { version = "1.21.2", features = ["full"] } diff --git a/index.html b/index.html index 56ff509..d43ef51 100644 --- a/index.html +++ b/index.html @@ -19,6 +19,10 @@ columns: 100; } + .error { + color: red; + } + .bisect-btn { width: 200px; height: 50px; @@ -76,66 +80,101 @@ impl<T> Struct<T> { diff --git a/src/bisect.rs b/src/bisect.rs index 1a76c54..a66c929 100644 --- a/src/bisect.rs +++ b/src/bisect.rs @@ -1,6 +1,7 @@ use std::process::Output; use std::{fs, process::Command, sync::mpsc}; +use chrono::{DateTime, Utc}; use color_eyre::eyre::{Context, ContextCompat}; use color_eyre::Result; use rusqlite::Connection; @@ -22,6 +23,7 @@ pub enum BisectStatus { pub struct Bisection { pub id: Uuid, pub code: String, + pub time: DateTime, pub status: BisectStatus, } @@ -83,6 +85,7 @@ pub fn process_job(job: Job, conn: &Connection) -> Result<()> { let mut bisect = Bisection { id: job.id, code: job.code.clone(), + time: Utc::now(), status: BisectStatus::InProgress, }; @@ -169,7 +172,7 @@ fn run_bisect_for_file(input: String, options: &Options) -> Result<(Output, JobS bisect .arg("--regress") - .arg(options.kind.as_deref().unwrap_or("ice")); // FIXME Make this configurable + .arg(options.kind.as_deref().unwrap_or("ice")); bisect.env("RUST_LOG", "error"); // overwrite RUST_LOG diff --git a/src/db.rs b/src/db.rs index d0530e5..3d112dc 100644 --- a/src/db.rs +++ b/src/db.rs @@ -11,6 +11,7 @@ pub fn setup(conn: &Connection) -> color_eyre::Result<()> { job_id STRING PRIMARY KEY, code STRING NOT NULL, status INTEGER NOT NULL DEFAULT 0, + time TIME NOT NULL, stdout_stderr STRING -- stdout or stderr depending on the status )", (), @@ -34,8 +35,8 @@ pub fn add_bisection(conn: &Connection, bisect: &Bisection) -> color_eyre::Resul let (status, stdout_stderr) = status_to_sql(&bisect.status); conn.execute( - "INSERT INTO bisect (job_id, code, status, stdout_stderr) VALUES (?1, ?2, ?3, ?4)", - (bisect.id, &bisect.code, status, stdout_stderr), + "INSERT INTO bisect (job_id, code, status, time, stdout_stderr) VALUES (?1, ?2, ?3, ?4, ?5)", + (bisect.id, &bisect.code, status, &bisect.time, stdout_stderr), ) .wrap_err("insert into database") .map(drop) @@ -54,7 +55,7 @@ pub fn update_bisection_status(conn: &Connection, bisect: &Bisection) -> color_e pub fn get_bisections(conn: &Connection) -> color_eyre::Result> { let mut select = conn - .prepare("SELECT job_id, code, status, stdout_stderr FROM bisect") + .prepare("SELECT job_id, code, status, time, stdout_stderr FROM bisect") .wrap_err("preparing select")?; let iter = select @@ -65,13 +66,14 @@ pub fn get_bisections(conn: &Connection) -> color_eyre::Result> { status: match row.get(2)? { 0 => BisectStatus::InProgress, 1 => BisectStatus::Error { - output: row.get(3)?, + output: row.get(4)?, }, 2 => BisectStatus::Success { - output: row.get(3)?, + output: row.get(4)?, }, _ => return Err(rusqlite::Error::InvalidQuery), // actually not lol }, + time: row.get(3)?, }) }) .wrap_err("getting bisections from db query")?; @@ -82,7 +84,7 @@ pub fn get_bisections(conn: &Connection) -> color_eyre::Result> { pub fn get_bisection(conn: &Connection, id: Uuid) -> color_eyre::Result> { let mut select = conn - .prepare("SELECT job_id, code, status, stdout_stderr FROM bisect WHERE job_id = ?1") + .prepare("SELECT job_id, code, status, time, stdout_stderr FROM bisect WHERE job_id = ?1") .wrap_err("preparing select")?; let mut iter = select @@ -93,13 +95,14 @@ pub fn get_bisection(conn: &Connection, id: Uuid) -> color_eyre::Result BisectStatus::InProgress, 1 => BisectStatus::Error { - output: row.get(3)?, + output: row.get(4)?, }, 2 => BisectStatus::Success { - output: row.get(3)?, + output: row.get(4)?, }, _ => return Err(rusqlite::Error::InvalidQuery), // actually not lol }, + time: row.get(3)?, }) }) .wrap_err("getting bisections from db query")?; From 80c6b2a8a273ffd1f4286e1133e1d9b77fc49120 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Thu, 29 Sep 2022 22:12:09 +0200 Subject: [PATCH 06/18] make into form --- index.html | 94 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/index.html b/index.html index d43ef51..21c88fa 100644 --- a/index.html +++ b/index.html @@ -32,12 +32,16 @@

Bisect rustc

- -
- - - - - - + > +
+ + -
-
+ + - + + + +
+
+ +

@@ -80,12 +86,8 @@ impl<T> Struct<T> { diff --git a/src/main.rs b/src/main.rs index daff821..340911e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,7 +22,7 @@ use tracing::{error, info, metadata::LevelFilter}; use tracing_subscriber::EnvFilter; use uuid::Uuid; -type SendChannel = Arc>>; +type SendChannel = Arc>>; type Conn = Arc>; #[tokio::main] @@ -37,7 +37,7 @@ async fn main() -> color_eyre::Result<()> { ) .init(); - let (job_queue_send, job_queue_recv) = mpsc::channel(); // FIXME: make this a sync_channel because bounds are cool + let (job_queue_send, job_queue_recv) = mpsc::sync_channel(10); let sqlite_db = env::var("SQLITE_DB").unwrap_or_else(|_| "bisect.sqlite".to_string()); @@ -62,7 +62,7 @@ async fn main() -> color_eyre::Result<()> { std::thread::spawn(|| bisect::bisect_worker(job_queue_recv, worker_conn)); - info!("Starting up server"); + info!("Starting up server on port 4000"); axum::Server::bind(&"0.0.0.0:4000".parse().unwrap()) .serve(app.into_make_service()) From 4bb561df9bfc2ee2a51b53c1388e5a6d83e8b894 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Sat, 26 Nov 2022 12:09:45 +0100 Subject: [PATCH 10/18] remove toolchains sometimes --- src/bisect.rs | 2 ++ src/main.rs | 3 ++ src/toolchain.rs | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 src/toolchain.rs diff --git a/src/bisect.rs b/src/bisect.rs index a66c929..1b3f043 100644 --- a/src/bisect.rs +++ b/src/bisect.rs @@ -107,6 +107,8 @@ pub fn process_job(job: Job, conn: &Connection) -> Result<()> { trace!(?bisect, "Finished bisection job"); + crate::toolchain::clean_toolchains()?; + Ok(()) } diff --git a/src/main.rs b/src/main.rs index 340911e..50ad1a3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ mod bisect; mod db; +mod toolchain; use std::sync::{mpsc, Arc, Mutex}; @@ -50,6 +51,8 @@ async fn main() -> color_eyre::Result<()> { db::setup(&worker_conn).wrap_err("db setup")?; + toolchain::clean_toolchains()?; + let app = Router::new() .route("/", get(|| async { index_html() })) .route("/bisect/:id", get(get_bisection)) diff --git a/src/toolchain.rs b/src/toolchain.rs new file mode 100644 index 0000000..97b0ae5 --- /dev/null +++ b/src/toolchain.rs @@ -0,0 +1,73 @@ +use std::process::Command; + +use color_eyre::{ + eyre::{eyre, Context}, + Result, +}; +use tracing::debug; + +const MAX_BISECTOR_TOOLCHAINS: usize = 15; + +#[tracing::instrument] +pub fn clean_toolchains() -> Result<()> { + let toolchains = get_toolchains()?; + let for_removal = filter_toolchain_for_removal(toolchains); + if !for_removal.is_empty() { + remove_toolchains(&for_removal)?; + } + + Ok(()) +} + +fn filter_toolchain_for_removal(mut toolchains: Vec) -> Vec { + toolchains.retain(|toolchain| toolchain.starts_with("bisector-")); + + let amount = toolchains.len(); + if amount <= MAX_BISECTOR_TOOLCHAINS { + debug!(%amount, "No toolchains removed"); + return Vec::new(); + } + + let to_remove = amount - MAX_BISECTOR_TOOLCHAINS; + + toolchains.into_iter().take(to_remove).collect() +} + +fn get_toolchains() -> Result> { + let mut command = Command::new("rustup"); + command.args(["toolchain", "list"]); + + let output = command + .output() + .wrap_err("running `rustup toolchain list`")?; + + if output.status.success() { + let stdout = + String::from_utf8(output.stdout).wrap_err("rustup returned non-utf-8 bytes")?; + + let toolchains = stdout.lines().map(ToOwned::to_owned).collect(); + + Ok(toolchains) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + Err(eyre!("`rustup toolchain list` failed").wrap_err(stderr)) + } +} + +fn remove_toolchains(toolchains: &[String]) -> Result<()> { + debug!(?toolchains, "Removing toolchains"); + let mut command = Command::new("rustup"); + command.args(["toolchain", "remove"]); + command.args(toolchains); + + let output = command + .output() + .wrap_err("running `rustup toolchain remove`")?; + + if output.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + Err(eyre!("`rustup toolchain remove` failed").wrap_err(stderr)) +} From 9a2f8ecdbc058e3e4c794f2bfbf7994626c38e8c Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Sat, 26 Nov 2022 12:17:37 +0100 Subject: [PATCH 11/18] add github link --- index.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/index.html b/index.html index 93abc24..47efd17 100644 --- a/index.html +++ b/index.html @@ -84,6 +84,13 @@ impl<T> Struct<T> { readonly > +
+ Made by Nilstrieb + Github +
+