download package (badly)

Cargo.lock

@@ -17,15 +17,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
-[[package]]
-name = "aho-corasick"
-version = "0.7.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "ansi_term"
 version = "0.12.1"
@@ -85,6 +76,12 @@ version = "3.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
 
+[[package]]
+name = "bytecount"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72feb31ffc86498dacdbd0fcebb56138e7177a8cc5cea4516031d15ae85a742e"
+
 [[package]]
 name = "bytes"
 version = "1.1.0"
@@ -185,6 +182,15 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
 
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.31"
@@ -213,6 +219,30 @@ dependencies = [
  "instant",
 ]
 
+[[package]]
+name = "filetime"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "winapi",
+]
+
+[[package]]
+name = "flate2"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af"
+dependencies = [
+ "cfg-if",
+ "crc32fast",
+ "libc",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -259,12 +289,6 @@ version = "0.3.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
 
-[[package]]
-name = "futures-io"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
-
 [[package]]
 name = "futures-sink"
 version = "0.3.21"
@@ -284,12 +308,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
 dependencies = [
  "futures-core",
- "futures-io",
  "futures-task",
- "memchr",
  "pin-project-lite",
  "pin-utils",
- "slab",
 ]
 
 [[package]]
@@ -492,6 +513,16 @@ version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b"
 
+[[package]]
+name = "lock_api"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
 [[package]]
 name = "log"
 version = "0.4.17"
@@ -522,12 +553,40 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
+[[package]]
+name = "miette"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd2adcfcced5d625bf90a958a82ae5b93231f57f3df1383fee28c9b5096d35ed"
+dependencies = [
+ "miette-derive",
+ "once_cell",
+ "thiserror",
+]
+
+[[package]]
+name = "miette-derive"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c01a8b61312d367ce87956bb686731f87e4c6dd5dbc550e8f06e3c24fb1f67f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.5.1"
@@ -571,19 +630,46 @@ dependencies = [
 name = "node-package-manager"
 version = "0.1.0"
 dependencies = [
+ "bytes",
  "clap",
  "color-eyre",
+ "flate2",
  "indexmap",
+ "node-semver",
  "reqwest",
- "semver_rs",
  "serde",
  "serde_json",
+ "tar",
+ "tokio",
  "tracing",
  "tracing-forest",
  "tracing-subscriber",
  "tracing-tree",
 ]
 
+[[package]]
+name = "node-semver"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8173fd025860308645e5cc6d9d75e23317b03b8a71216d3fc78e375eba386d9"
+dependencies = [
+ "bytecount",
+ "miette",
+ "nom",
+ "serde",
+ "thiserror",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.13.1"
@@ -666,6 +752,29 @@ version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b"
 
+[[package]]
+name = "parking_lot"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-sys",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.1.0"
@@ -747,8 +856,6 @@ version = "1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
 dependencies = [
- "aho-corasick",
- "memchr",
  "regex-syntax",
 ]
 
@@ -875,6 +982,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
 [[package]]
 name = "sct"
 version = "0.7.0"
@@ -908,19 +1021,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "semver_rs"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9594e1aab972e5b5ecbb330754bef51c7ba0dc12644b6bae9e09a4e19d472586"
-dependencies = [
- "lazy_static",
- "regex",
- "serde",
- "thiserror",
- "unicase",
-]
-
 [[package]]
 name = "serde"
 version = "1.0.137"
@@ -973,6 +1073,15 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.6"
@@ -1018,6 +1127,17 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "tar"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.3.0"
@@ -1103,11 +1223,25 @@ dependencies = [
  "mio",
  "num_cpus",
  "once_cell",
+ "parking_lot",
  "pin-project-lite",
+ "signal-hook-registry",
  "socket2",
+ "tokio-macros",
  "winapi",
 ]
 
+[[package]]
+name = "tokio-macros"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tokio-native-tls"
 version = "0.3.0"
@@ -1252,15 +1386,6 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
-[[package]]
-name = "unicase"
-version = "2.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
-dependencies = [
- "version_check",
-]
-
 [[package]]
 name = "unicode-bidi"
 version = "0.3.8"
@@ -1511,3 +1636,12 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
 dependencies = [
  "winapi",
 ]
+
+[[package]]
+name = "xattr"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc"
+dependencies = [
+ "libc",
+]

Cargo.toml

@@ -6,13 +6,17 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+bytes = "1.1.0"
 clap = { version = "3.1.18", features = ["derive"] }
 color-eyre = "0.6.1"
-indexmap = { version = "1.8.1", features = ["serde"] }
+flate2 = "1.0.23"
-reqwest = { version = "0.11.10", features = ["blocking", "rustls-tls", "json"] }
+indexmap = { version = "1.8.1", features = ["serde", "std"] }
-semver_rs = { version = "0.2.0", features = ["serde"] }
+node-semver = "2.0.0"
+reqwest = { version = "0.11.10", features = ["rustls-tls", "json"] }
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
+tar = "0.4.38"
+tokio = { version = "1.18.2", features = ["full"] }
 tracing = "0.1.34"
 tracing-forest = { version = "0.1.4", features = ["env-filter"] }
 tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }

src/download.rs

@@ -1,14 +1,19 @@
 #![allow(dead_code)]
 
+use std::path::Path;
+
+use bytes::Buf;
 use color_eyre::Result;
 use indexmap::IndexMap;
-use reqwest::blocking::Client;
+use node_semver::Version;
+use reqwest::Client;
 use serde::Deserialize;
-use tracing::debug;
+use tar::Archive;
+use tracing::{debug, info};
 
 use crate::{
     manifest::{Bugs, Human, Person, Repository},
-    PackageJson,
+    PackageJson, WrapErr,
 };
 
 #[derive(Debug, Deserialize)]
@@ -53,8 +58,7 @@ pub struct PackageMeta {
     pub name: String,
     pub time: IndexMap<String, String>,
     pub users: IndexMap<String, bool>,
-    pub versions: IndexMap<String, VersionMeta>,
+    pub versions: IndexMap<Version, VersionMeta>,
-
     pub author: Human,
     pub bugs: Option<Bugs>,
     pub contributors: Option<Vec<Human>>,
@@ -69,6 +73,7 @@ pub struct PackageMeta {
     pub repository: Option<Repository>,
 }
 
+#[derive(Default)]
 pub struct NpmClient {
     reqwest: Client,
 }
@@ -82,13 +87,38 @@ impl NpmClient {
     }
 
     #[tracing::instrument(skip(self))]
-    pub fn inspect_package(&self, name: &str) -> Result<PackageMeta> {
+    pub async fn fetch_package_meta(&self, name: &str) -> Result<PackageMeta> {
-        let res = self.reqwest.get(format!("{BASE_URL}/{name}")).send()?;
+        let res = self
+            .reqwest
+            .get(format!("{BASE_URL}/{name}"))
+            .send()
+            .await?;
         let code = res.status();
-        let body = res.text()?;
+        let body = res.text().await?;
         let meta = serde_json::from_str::<PackageMeta>(&body)?;
 
         debug!(?code, ?meta, "Received response");
         Ok(meta)
     }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn download_package(&self, name: &str, url: &str) -> Result<()> {
+        let response = self
+            .reqwest
+            .get(url)
+            .send()
+            .await
+            .wrap_err("getting response")?;
+        let tarball = response.bytes().await.wrap_err("fetching body")?;
+
+        let tar = flate2::read::GzDecoder::new(tarball.reader());
+        let mut archive = tar::Archive::new(tar);
+        archive
+            .unpack(Path::new("node_modules").join(name))
+            .wrap_err("unpack tarball")?;
+
+        info!("successfully downloaded package");
+
+        Ok(())
+    }
 }

src/main.rs

@@ -1,61 +1,43 @@
-use std::fs;
+use std::{fs, io};
 
 use color_eyre::{
     eyre::{bail, WrapErr},
     Result,
 };
-use semver_rs::{Range, Version};
+use tracing::metadata::LevelFilter;
-use tracing::{debug, info, metadata::LevelFilter, warn};
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
 
-use crate::{download::NpmClient, manifest::PackageJson};
+use crate::{download::NpmClient, manifest::PackageJson, resolve::ResolveContext};
 
 mod download;
 mod manifest;
+mod resolve;
 
-fn main() -> Result<()> {
+#[tokio::main]
+async fn main() -> Result<()> {
     color_eyre::install()?;
     setup_tracing()?;
 
-    let manifest = "testing/package.json";
+    let manifest = "package.json";
     let manifest = fs::read_to_string(manifest).wrap_err("Opening package.json file")?;
 
     let manifest: PackageJson = serde_json::from_str(&manifest)?;
 
-    debug!(?manifest, "Read manifest");
+    let resolve_context = ResolveContext::new();
 
-    let client = NpmClient::new();
+    match fs::metadata("node_modules") {
-
+        Ok(_) => {}
-    for (name, requested_version) in &manifest.dependencies.unwrap() {
+        Err(e) if e.kind() == io::ErrorKind::NotFound => {
-        look_at_package(name, requested_version, &client).wrap_err(format!("package {name}"))?;
+            fs::create_dir("node_modules").wrap_err("creating node_modules directory")?;
+        }
+        Err(e) => bail!(e),
     }
 
-    Ok(())
+    for (name, requested_version) in &manifest.dependencies.unwrap() {
-}
+        resolve_context
-
+            .download_package_and_deps(name, requested_version)
-#[tracing::instrument(skip(client))]
+            .await
-fn look_at_package(name: &str, requested_version: &str, client: &NpmClient) -> Result<()> {
+            .wrap_err(format!("package {name}"))?;
-    let requested = Range::new(requested_version).parse()?;
-
-    let meta = client.inspect_package(name)?;
-
-    info!(versions = ?meta.versions.keys());
-
-    let mut versions = meta
-        .versions
-        .keys()
-        .map(|v| Ok((v, Version::new(v).parse()?)))
-        .collect::<Result<Vec<_>, semver_rs::Error>>()?;
-
-    versions.sort_by(|a, b| b.cmp(a));
-
-    let chosen = versions.iter().find(|(_, version)| requested.test(version));
-
-    match chosen {
-        Some((version, _)) => {
-            info!(?version, "Found version")
-        }
-        None => bail!("could not find matching version for '{requested_version}'"),
     }
 
     Ok(())

src/manifest.rs

@@ -1,4 +1,5 @@
 use indexmap::map::IndexMap;
+use node_semver::{Range, Version};
 use serde::Deserialize;
 
 #[derive(Debug, Deserialize)]
@@ -67,12 +68,12 @@ pub enum Override {
     Nested(IndexMap<String, Override>),
 }
 
-/// https://docs.npmjs.com/cli/v8/configuring-npm/package-json
+/// <https://docs.npmjs.com/cli/v8/configuring-npm/package-json>
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct PackageJson {
     pub name: String,
-    pub version: String,
+    pub version: Version,
     pub description: Option<String>,
     pub keywords: Option<Vec<String>>,
     pub homepage: Option<String>,
@@ -90,12 +91,12 @@ pub struct PackageJson {
     pub repository: Option<Repository>,
     pub scripts: Option<IndexMap<String, String>>,
     pub config: Option<IndexMap<String, serde_json::Value>>,
-    pub dependencies: Option<IndexMap<String, String>>,
+    pub dependencies: Option<IndexMap<String, Range>>,
-    pub dev_dependencies: Option<IndexMap<String, String>>,
+    pub dev_dependencies: Option<IndexMap<String, Range>>,
-    pub peer_dependencies: Option<IndexMap<String, String>>,
+    pub peer_dependencies: Option<IndexMap<String, Range>>,
     pub peer_dependencies_meta: Option<IndexMap<String, PeerDependencyMeta>>,
     pub bundled_dependencies: Option<Vec<String>>,
-    pub optional_dependencies: Option<IndexMap<String, String>>,
+    pub optional_dependencies: Option<IndexMap<String, Range>>,
     pub overrides: Option<IndexMap<String, Override>>,
     pub engines: Option<IndexMap<String, String>>,
     pub os: Option<Vec<String>>,

src/resolve.rs

@@ -0,0 +1,79 @@
+use std::{
+    collections::HashMap,
+    sync::{Arc, RwLock},
+};
+
+use color_eyre::{eyre::bail, Result};
+use node_semver::Range;
+use tracing::{debug, info};
+
+use crate::{download::PackageMeta, NpmClient, WrapErr};
+
+pub struct ResolveContext {
+    meta_cache: Arc<RwLock<HashMap<String, Arc<PackageMeta>>>>,
+    client: NpmClient,
+}
+
+impl ResolveContext {
+    pub fn new() -> Self {
+        Self {
+            meta_cache: Arc::new(Default::default()),
+            client: NpmClient::default(),
+        }
+    }
+
+    async fn get_meta(&self, name: &str) -> Result<Arc<PackageMeta>> {
+        {
+            let cache_read = self.meta_cache.read().unwrap();
+            if let Some(meta) = cache_read.get(name) {
+                return Ok(Arc::clone(meta));
+            }
+        }
+
+        debug!("Fetching package info..");
+
+        // two futures might race here - who cares
+        let meta = self
+            .client
+            .fetch_package_meta(name)
+            .await
+            .wrap_err("fetching package metadata")?;
+
+        let meta = Arc::new(meta);
+
+        let mut cache_write = self.meta_cache.write().unwrap();
+        cache_write.insert(name.to_owned(), Arc::clone(&meta));
+
+        Ok(meta)
+    }
+
+    #[tracing::instrument(skip(self, requested_version), fields(requested_version = %requested_version))]
+    pub async fn download_package_and_deps(
+        &self,
+        name: &str,
+        requested_version: &Range,
+    ) -> Result<()> {
+        let meta = self.get_meta(name).await?;
+
+        info!(versions = ?meta.versions.keys().map(ToString::to_string).collect::<Vec<_>>());
+
+        let chosen = meta
+            .versions
+            .keys()
+            .filter(|version| version.satisfies(requested_version))
+            .max();
+
+        match chosen {
+            Some(version) => {
+                info!(%version, "Found version");
+                self.client
+                    .download_package(name, &meta.versions[version].dist.tarball)
+                    .await
+                    .wrap_err("downloading package")?;
+            }
+            None => bail!("could not find matching version for '{requested_version}'"),
+        }
+
+        Ok(())
+    }
+}