diff --git a/Cargo.lock b/Cargo.lock index a7535d6..04efb0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,15 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aho-corasick" -version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" -dependencies = [ - "memchr", -] - [[package]] name = "ansi_term" version = "0.12.1" @@ -85,6 +76,12 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +[[package]] +name = "bytecount" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72feb31ffc86498dacdbd0fcebb56138e7177a8cc5cea4516031d15ae85a742e" + [[package]] name = "bytes" version = "1.1.0" @@ -185,6 +182,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "encoding_rs" version = "0.8.31" @@ -213,6 +219,30 @@ dependencies = [ "instant", ] +[[package]] +name = "filetime" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi", +] + +[[package]] +name = "flate2" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -259,12 +289,6 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" -[[package]] -name = "futures-io" -version = "0.3.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" - [[package]] name = "futures-sink" version = "0.3.21" @@ -284,12 +308,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ "futures-core", - "futures-io", "futures-task", - "memchr", "pin-project-lite", "pin-utils", - "slab", ] [[package]] @@ -492,6 +513,16 @@ version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +[[package]] +name = "lock_api" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -522,12 +553,40 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "miette" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd2adcfcced5d625bf90a958a82ae5b93231f57f3df1383fee28c9b5096d35ed" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", +] + +[[package]] +name = "miette-derive" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c01a8b61312d367ce87956bb686731f87e4c6dd5dbc550e8f06e3c24fb1f67f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "mime" version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.5.1" @@ -571,19 +630,46 @@ dependencies = [ name = "node-package-manager" version = "0.1.0" dependencies = [ + "bytes", "clap", "color-eyre", + "flate2", "indexmap", + "node-semver", "reqwest", - "semver_rs", "serde", "serde_json", + "tar", + "tokio", "tracing", "tracing-forest", "tracing-subscriber", "tracing-tree", ] +[[package]] +name = "node-semver" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8173fd025860308645e5cc6d9d75e23317b03b8a71216d3fc78e375eba386d9" +dependencies = [ + "bytecount", + "miette", + "nom", + "serde", + "thiserror", +] + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -666,6 +752,29 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b" +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -747,8 +856,6 @@ version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" dependencies = [ - "aho-corasick", - "memchr", "regex-syntax", ] @@ -875,6 +982,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "sct" version = "0.7.0" @@ -908,19 +1021,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver_rs" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9594e1aab972e5b5ecbb330754bef51c7ba0dc12644b6bae9e09a4e19d472586" -dependencies = [ - "lazy_static", - "regex", - "serde", - "thiserror", - "unicase", -] - [[package]] name = "serde" version = "1.0.137" @@ -973,6 +1073,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.6" @@ -1018,6 +1127,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -1103,11 +1223,25 @@ dependencies = [ "mio", "num_cpus", "once_cell", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", "winapi", ] +[[package]] +name = "tokio-macros" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-native-tls" version = "0.3.0" @@ -1252,15 +1386,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" -[[package]] -name = "unicase" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.8" @@ -1511,3 +1636,12 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index 1e2277f..f240856 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,13 +6,17 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +bytes = "1.1.0" clap = { version = "3.1.18", features = ["derive"] } color-eyre = "0.6.1" -indexmap = { version = "1.8.1", features = ["serde"] } -reqwest = { version = "0.11.10", features = ["blocking", "rustls-tls", "json"] } -semver_rs = { version = "0.2.0", features = ["serde"] } +flate2 = "1.0.23" +indexmap = { version = "1.8.1", features = ["serde", "std"] } +node-semver = "2.0.0" +reqwest = { version = "0.11.10", features = ["rustls-tls", "json"] } serde = { version = "1.0.137", features = ["derive"] } serde_json = "1.0.81" +tar = "0.4.38" +tokio = { version = "1.18.2", features = ["full"] } tracing = "0.1.34" tracing-forest = { version = "0.1.4", features = ["env-filter"] } tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } diff --git a/src/download.rs b/src/download.rs index a7cfbaf..2877913 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,14 +1,19 @@ #![allow(dead_code)] +use std::path::Path; + +use bytes::Buf; use color_eyre::Result; use indexmap::IndexMap; -use reqwest::blocking::Client; +use node_semver::Version; +use reqwest::Client; use serde::Deserialize; -use tracing::debug; +use tar::Archive; +use tracing::{debug, info}; use crate::{ manifest::{Bugs, Human, Person, Repository}, - PackageJson, + PackageJson, WrapErr, }; #[derive(Debug, Deserialize)] @@ -53,8 +58,7 @@ pub struct PackageMeta { pub name: String, pub time: IndexMap, pub users: IndexMap, - pub versions: IndexMap, - + pub versions: IndexMap, pub author: Human, pub bugs: Option, pub contributors: Option>, @@ -69,6 +73,7 @@ pub struct PackageMeta { pub repository: Option, } +#[derive(Default)] pub struct NpmClient { reqwest: Client, } @@ -82,13 +87,38 @@ impl NpmClient { } #[tracing::instrument(skip(self))] - pub fn inspect_package(&self, name: &str) -> Result { - let res = self.reqwest.get(format!("{BASE_URL}/{name}")).send()?; + pub async fn fetch_package_meta(&self, name: &str) -> Result { + let res = self + .reqwest + .get(format!("{BASE_URL}/{name}")) + .send() + .await?; let code = res.status(); - let body = res.text()?; + let body = res.text().await?; let meta = serde_json::from_str::(&body)?; debug!(?code, ?meta, "Received response"); Ok(meta) } + + #[tracing::instrument(skip(self))] + pub async fn download_package(&self, name: &str, url: &str) -> Result<()> { + let response = self + .reqwest + .get(url) + .send() + .await + .wrap_err("getting response")?; + let tarball = response.bytes().await.wrap_err("fetching body")?; + + let tar = flate2::read::GzDecoder::new(tarball.reader()); + let mut archive = tar::Archive::new(tar); + archive + .unpack(Path::new("node_modules").join(name)) + .wrap_err("unpack tarball")?; + + info!("successfully downloaded package"); + + Ok(()) + } } diff --git a/src/main.rs b/src/main.rs index 44c6c1a..a35ac5f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,61 +1,43 @@ -use std::fs; +use std::{fs, io}; use color_eyre::{ eyre::{bail, WrapErr}, Result, }; -use semver_rs::{Range, Version}; -use tracing::{debug, info, metadata::LevelFilter, warn}; +use tracing::metadata::LevelFilter; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry}; -use crate::{download::NpmClient, manifest::PackageJson}; +use crate::{download::NpmClient, manifest::PackageJson, resolve::ResolveContext}; mod download; mod manifest; +mod resolve; -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { color_eyre::install()?; setup_tracing()?; - let manifest = "testing/package.json"; + let manifest = "package.json"; let manifest = fs::read_to_string(manifest).wrap_err("Opening package.json file")?; let manifest: PackageJson = serde_json::from_str(&manifest)?; - debug!(?manifest, "Read manifest"); + let resolve_context = ResolveContext::new(); - let client = NpmClient::new(); - - for (name, requested_version) in &manifest.dependencies.unwrap() { - look_at_package(name, requested_version, &client).wrap_err(format!("package {name}"))?; + match fs::metadata("node_modules") { + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => { + fs::create_dir("node_modules").wrap_err("creating node_modules directory")?; + } + Err(e) => bail!(e), } - Ok(()) -} - -#[tracing::instrument(skip(client))] -fn look_at_package(name: &str, requested_version: &str, client: &NpmClient) -> Result<()> { - let requested = Range::new(requested_version).parse()?; - - let meta = client.inspect_package(name)?; - - info!(versions = ?meta.versions.keys()); - - let mut versions = meta - .versions - .keys() - .map(|v| Ok((v, Version::new(v).parse()?))) - .collect::, semver_rs::Error>>()?; - - versions.sort_by(|a, b| b.cmp(a)); - - let chosen = versions.iter().find(|(_, version)| requested.test(version)); - - match chosen { - Some((version, _)) => { - info!(?version, "Found version") - } - None => bail!("could not find matching version for '{requested_version}'"), + for (name, requested_version) in &manifest.dependencies.unwrap() { + resolve_context + .download_package_and_deps(name, requested_version) + .await + .wrap_err(format!("package {name}"))?; } Ok(()) diff --git a/src/manifest.rs b/src/manifest.rs index 0778a9c..0f8c895 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -1,4 +1,5 @@ use indexmap::map::IndexMap; +use node_semver::{Range, Version}; use serde::Deserialize; #[derive(Debug, Deserialize)] @@ -67,12 +68,12 @@ pub enum Override { Nested(IndexMap), } -/// https://docs.npmjs.com/cli/v8/configuring-npm/package-json +/// #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PackageJson { pub name: String, - pub version: String, + pub version: Version, pub description: Option, pub keywords: Option>, pub homepage: Option, @@ -90,12 +91,12 @@ pub struct PackageJson { pub repository: Option, pub scripts: Option>, pub config: Option>, - pub dependencies: Option>, - pub dev_dependencies: Option>, - pub peer_dependencies: Option>, + pub dependencies: Option>, + pub dev_dependencies: Option>, + pub peer_dependencies: Option>, pub peer_dependencies_meta: Option>, pub bundled_dependencies: Option>, - pub optional_dependencies: Option>, + pub optional_dependencies: Option>, pub overrides: Option>, pub engines: Option>, pub os: Option>, diff --git a/src/resolve.rs b/src/resolve.rs new file mode 100644 index 0000000..df898a0 --- /dev/null +++ b/src/resolve.rs @@ -0,0 +1,79 @@ +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use color_eyre::{eyre::bail, Result}; +use node_semver::Range; +use tracing::{debug, info}; + +use crate::{download::PackageMeta, NpmClient, WrapErr}; + +pub struct ResolveContext { + meta_cache: Arc>>>, + client: NpmClient, +} + +impl ResolveContext { + pub fn new() -> Self { + Self { + meta_cache: Arc::new(Default::default()), + client: NpmClient::default(), + } + } + + async fn get_meta(&self, name: &str) -> Result> { + { + let cache_read = self.meta_cache.read().unwrap(); + if let Some(meta) = cache_read.get(name) { + return Ok(Arc::clone(meta)); + } + } + + debug!("Fetching package info.."); + + // two futures might race here - who cares + let meta = self + .client + .fetch_package_meta(name) + .await + .wrap_err("fetching package metadata")?; + + let meta = Arc::new(meta); + + let mut cache_write = self.meta_cache.write().unwrap(); + cache_write.insert(name.to_owned(), Arc::clone(&meta)); + + Ok(meta) + } + + #[tracing::instrument(skip(self, requested_version), fields(requested_version = %requested_version))] + pub async fn download_package_and_deps( + &self, + name: &str, + requested_version: &Range, + ) -> Result<()> { + let meta = self.get_meta(name).await?; + + info!(versions = ?meta.versions.keys().map(ToString::to_string).collect::>()); + + let chosen = meta + .versions + .keys() + .filter(|version| version.satisfies(requested_version)) + .max(); + + match chosen { + Some(version) => { + info!(%version, "Found version"); + self.client + .download_package(name, &meta.versions[version].dist.tarball) + .await + .wrap_err("downloading package")?; + } + None => bail!("could not find matching version for '{requested_version}'"), + } + + Ok(()) + } +}