download package (badly)

This commit is contained in:
nora 2022-05-15 14:07:52 +02:00
parent b4bc82f7e4
commit f9c5a18720
6 changed files with 327 additions and 97 deletions

220
Cargo.lock generated
View file

@ -17,15 +17,6 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aho-corasick"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
dependencies = [
"memchr",
]
[[package]]
name = "ansi_term"
version = "0.12.1"
@ -85,6 +76,12 @@ version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "bytecount"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72feb31ffc86498dacdbd0fcebb56138e7177a8cc5cea4516031d15ae85a742e"
[[package]]
name = "bytes"
version = "1.1.0"
@ -185,6 +182,15 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
]
[[package]]
name = "encoding_rs"
version = "0.8.31"
@ -213,6 +219,30 @@ dependencies = [
"instant",
]
[[package]]
name = "filetime"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"winapi",
]
[[package]]
name = "flate2"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af"
dependencies = [
"cfg-if",
"crc32fast",
"libc",
"miniz_oxide",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -259,12 +289,6 @@ version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
[[package]]
name = "futures-io"
version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
[[package]]
name = "futures-sink"
version = "0.3.21"
@ -284,12 +308,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
dependencies = [
"futures-core",
"futures-io",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]]
@ -492,6 +513,16 @@ version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b"
[[package]]
name = "lock_api"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.17"
@ -522,12 +553,40 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "miette"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd2adcfcced5d625bf90a958a82ae5b93231f57f3df1383fee28c9b5096d35ed"
dependencies = [
"miette-derive",
"once_cell",
"thiserror",
]
[[package]]
name = "miette-derive"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c01a8b61312d367ce87956bb686731f87e4c6dd5dbc550e8f06e3c24fb1f67f"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.5.1"
@ -571,19 +630,46 @@ dependencies = [
name = "node-package-manager"
version = "0.1.0"
dependencies = [
"bytes",
"clap",
"color-eyre",
"flate2",
"indexmap",
"node-semver",
"reqwest",
"semver_rs",
"serde",
"serde_json",
"tar",
"tokio",
"tracing",
"tracing-forest",
"tracing-subscriber",
"tracing-tree",
]
[[package]]
name = "node-semver"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8173fd025860308645e5cc6d9d75e23317b03b8a71216d3fc78e375eba386d9"
dependencies = [
"bytecount",
"miette",
"nom",
"serde",
"thiserror",
]
[[package]]
name = "nom"
version = "7.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "num_cpus"
version = "1.13.1"
@ -666,6 +752,29 @@ version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b"
[[package]]
name = "parking_lot"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-sys",
]
[[package]]
name = "percent-encoding"
version = "2.1.0"
@ -747,8 +856,6 @@ version = "1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
@ -875,6 +982,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.7.0"
@ -908,19 +1021,6 @@ dependencies = [
"libc",
]
[[package]]
name = "semver_rs"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9594e1aab972e5b5ecbb330754bef51c7ba0dc12644b6bae9e09a4e19d472586"
dependencies = [
"lazy_static",
"regex",
"serde",
"thiserror",
"unicase",
]
[[package]]
name = "serde"
version = "1.0.137"
@ -973,6 +1073,15 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
dependencies = [
"libc",
]
[[package]]
name = "slab"
version = "0.4.6"
@ -1018,6 +1127,17 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "tar"
version = "0.4.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6"
dependencies = [
"filetime",
"libc",
"xattr",
]
[[package]]
name = "tempfile"
version = "3.3.0"
@ -1103,11 +1223,25 @@ dependencies = [
"mio",
"num_cpus",
"once_cell",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"winapi",
]
[[package]]
name = "tokio-macros"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tokio-native-tls"
version = "0.3.0"
@ -1252,15 +1386,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
version = "0.3.8"
@ -1511,3 +1636,12 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
dependencies = [
"winapi",
]
[[package]]
name = "xattr"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc"
dependencies = [
"libc",
]

View file

@ -6,13 +6,17 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bytes = "1.1.0"
clap = { version = "3.1.18", features = ["derive"] }
color-eyre = "0.6.1"
indexmap = { version = "1.8.1", features = ["serde"] }
reqwest = { version = "0.11.10", features = ["blocking", "rustls-tls", "json"] }
semver_rs = { version = "0.2.0", features = ["serde"] }
flate2 = "1.0.23"
indexmap = { version = "1.8.1", features = ["serde", "std"] }
node-semver = "2.0.0"
reqwest = { version = "0.11.10", features = ["rustls-tls", "json"] }
serde = { version = "1.0.137", features = ["derive"] }
serde_json = "1.0.81"
tar = "0.4.38"
tokio = { version = "1.18.2", features = ["full"] }
tracing = "0.1.34"
tracing-forest = { version = "0.1.4", features = ["env-filter"] }
tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }

View file

@ -1,14 +1,19 @@
#![allow(dead_code)]
use std::path::Path;
use bytes::Buf;
use color_eyre::Result;
use indexmap::IndexMap;
use reqwest::blocking::Client;
use node_semver::Version;
use reqwest::Client;
use serde::Deserialize;
use tracing::debug;
use tar::Archive;
use tracing::{debug, info};
use crate::{
manifest::{Bugs, Human, Person, Repository},
PackageJson,
PackageJson, WrapErr,
};
#[derive(Debug, Deserialize)]
@ -53,8 +58,7 @@ pub struct PackageMeta {
pub name: String,
pub time: IndexMap<String, String>,
pub users: IndexMap<String, bool>,
pub versions: IndexMap<String, VersionMeta>,
pub versions: IndexMap<Version, VersionMeta>,
pub author: Human,
pub bugs: Option<Bugs>,
pub contributors: Option<Vec<Human>>,
@ -69,6 +73,7 @@ pub struct PackageMeta {
pub repository: Option<Repository>,
}
#[derive(Default)]
pub struct NpmClient {
reqwest: Client,
}
@ -82,13 +87,38 @@ impl NpmClient {
}
#[tracing::instrument(skip(self))]
pub fn inspect_package(&self, name: &str) -> Result<PackageMeta> {
let res = self.reqwest.get(format!("{BASE_URL}/{name}")).send()?;
pub async fn fetch_package_meta(&self, name: &str) -> Result<PackageMeta> {
let res = self
.reqwest
.get(format!("{BASE_URL}/{name}"))
.send()
.await?;
let code = res.status();
let body = res.text()?;
let body = res.text().await?;
let meta = serde_json::from_str::<PackageMeta>(&body)?;
debug!(?code, ?meta, "Received response");
Ok(meta)
}
#[tracing::instrument(skip(self))]
pub async fn download_package(&self, name: &str, url: &str) -> Result<()> {
let response = self
.reqwest
.get(url)
.send()
.await
.wrap_err("getting response")?;
let tarball = response.bytes().await.wrap_err("fetching body")?;
let tar = flate2::read::GzDecoder::new(tarball.reader());
let mut archive = tar::Archive::new(tar);
archive
.unpack(Path::new("node_modules").join(name))
.wrap_err("unpack tarball")?;
info!("successfully downloaded package");
Ok(())
}
}

View file

@ -1,61 +1,43 @@
use std::fs;
use std::{fs, io};
use color_eyre::{
eyre::{bail, WrapErr},
Result,
};
use semver_rs::{Range, Version};
use tracing::{debug, info, metadata::LevelFilter, warn};
use tracing::metadata::LevelFilter;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
use crate::{download::NpmClient, manifest::PackageJson};
use crate::{download::NpmClient, manifest::PackageJson, resolve::ResolveContext};
mod download;
mod manifest;
mod resolve;
fn main() -> Result<()> {
#[tokio::main]
async fn main() -> Result<()> {
color_eyre::install()?;
setup_tracing()?;
let manifest = "testing/package.json";
let manifest = "package.json";
let manifest = fs::read_to_string(manifest).wrap_err("Opening package.json file")?;
let manifest: PackageJson = serde_json::from_str(&manifest)?;
debug!(?manifest, "Read manifest");
let resolve_context = ResolveContext::new();
let client = NpmClient::new();
match fs::metadata("node_modules") {
Ok(_) => {}
Err(e) if e.kind() == io::ErrorKind::NotFound => {
fs::create_dir("node_modules").wrap_err("creating node_modules directory")?;
}
Err(e) => bail!(e),
}
for (name, requested_version) in &manifest.dependencies.unwrap() {
look_at_package(name, requested_version, &client).wrap_err(format!("package {name}"))?;
}
Ok(())
}
#[tracing::instrument(skip(client))]
fn look_at_package(name: &str, requested_version: &str, client: &NpmClient) -> Result<()> {
let requested = Range::new(requested_version).parse()?;
let meta = client.inspect_package(name)?;
info!(versions = ?meta.versions.keys());
let mut versions = meta
.versions
.keys()
.map(|v| Ok((v, Version::new(v).parse()?)))
.collect::<Result<Vec<_>, semver_rs::Error>>()?;
versions.sort_by(|a, b| b.cmp(a));
let chosen = versions.iter().find(|(_, version)| requested.test(version));
match chosen {
Some((version, _)) => {
info!(?version, "Found version")
}
None => bail!("could not find matching version for '{requested_version}'"),
resolve_context
.download_package_and_deps(name, requested_version)
.await
.wrap_err(format!("package {name}"))?;
}
Ok(())

View file

@ -1,4 +1,5 @@
use indexmap::map::IndexMap;
use node_semver::{Range, Version};
use serde::Deserialize;
#[derive(Debug, Deserialize)]
@ -67,12 +68,12 @@ pub enum Override {
Nested(IndexMap<String, Override>),
}
/// https://docs.npmjs.com/cli/v8/configuring-npm/package-json
/// <https://docs.npmjs.com/cli/v8/configuring-npm/package-json>
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PackageJson {
pub name: String,
pub version: String,
pub version: Version,
pub description: Option<String>,
pub keywords: Option<Vec<String>>,
pub homepage: Option<String>,
@ -90,12 +91,12 @@ pub struct PackageJson {
pub repository: Option<Repository>,
pub scripts: Option<IndexMap<String, String>>,
pub config: Option<IndexMap<String, serde_json::Value>>,
pub dependencies: Option<IndexMap<String, String>>,
pub dev_dependencies: Option<IndexMap<String, String>>,
pub peer_dependencies: Option<IndexMap<String, String>>,
pub dependencies: Option<IndexMap<String, Range>>,
pub dev_dependencies: Option<IndexMap<String, Range>>,
pub peer_dependencies: Option<IndexMap<String, Range>>,
pub peer_dependencies_meta: Option<IndexMap<String, PeerDependencyMeta>>,
pub bundled_dependencies: Option<Vec<String>>,
pub optional_dependencies: Option<IndexMap<String, String>>,
pub optional_dependencies: Option<IndexMap<String, Range>>,
pub overrides: Option<IndexMap<String, Override>>,
pub engines: Option<IndexMap<String, String>>,
pub os: Option<Vec<String>>,

79
src/resolve.rs Normal file
View file

@ -0,0 +1,79 @@
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use color_eyre::{eyre::bail, Result};
use node_semver::Range;
use tracing::{debug, info};
use crate::{download::PackageMeta, NpmClient, WrapErr};
pub struct ResolveContext {
meta_cache: Arc<RwLock<HashMap<String, Arc<PackageMeta>>>>,
client: NpmClient,
}
impl ResolveContext {
pub fn new() -> Self {
Self {
meta_cache: Arc::new(Default::default()),
client: NpmClient::default(),
}
}
async fn get_meta(&self, name: &str) -> Result<Arc<PackageMeta>> {
{
let cache_read = self.meta_cache.read().unwrap();
if let Some(meta) = cache_read.get(name) {
return Ok(Arc::clone(meta));
}
}
debug!("Fetching package info..");
// two futures might race here - who cares
let meta = self
.client
.fetch_package_meta(name)
.await
.wrap_err("fetching package metadata")?;
let meta = Arc::new(meta);
let mut cache_write = self.meta_cache.write().unwrap();
cache_write.insert(name.to_owned(), Arc::clone(&meta));
Ok(meta)
}
#[tracing::instrument(skip(self, requested_version), fields(requested_version = %requested_version))]
pub async fn download_package_and_deps(
&self,
name: &str,
requested_version: &Range,
) -> Result<()> {
let meta = self.get_meta(name).await?;
info!(versions = ?meta.versions.keys().map(ToString::to_string).collect::<Vec<_>>());
let chosen = meta
.versions
.keys()
.filter(|version| version.satisfies(requested_version))
.max();
match chosen {
Some(version) => {
info!(%version, "Found version");
self.client
.download_package(name, &meta.versions[version].dist.tarball)
.await
.wrap_err("downloading package")?;
}
None => bail!("could not find matching version for '{requested_version}'"),
}
Ok(())
}
}