Skip to content

Commit 0ada18d

Browse files
borsbors
committed
Auto merge of #656 - Mark-Simulacrum:bump-deps, r=Mark-Simulacrum
Bump dependencies
2 parents b219427 + cdbc06e commit 0ada18d

File tree

10 files changed

+989
-792
lines changed

10 files changed

+989
-792
lines changed

‎Cargo.lock

Lines changed: 839 additions & 730 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,14 @@ opt-level = 0
1515
[dependencies]
1616
base64 = "0.13.0"
1717
bytes = "0.4.9"
18+
bytes_1 = { version = "1", package = "bytes" }
1819
chrono = { version = "0.4", features = ["serde"] }
19-
crates-index = "0.16.2"
20+
crates-index = "0.18"
2021
crossbeam-utils = "0.8"
2122
crossbeam-channel = "0.5"
2223
csv = "1.0.2"
2324
docsrs-metadata = { git = "https://github.com/rust-lang/docs.rs/" }
24-
dotenv = "0.13"
25+
dotenv = "0.15"
2526
failure = "0.1.3"
2627
flate2 = "1"
2728
http = "0.1.10"
@@ -30,9 +31,9 @@ lazy_static = "1.0"
3031
mime = "0.3.1"
3132
minifier = { version = "0.0.20", features = ["html"] }
3233
r2d2 = "0.8.2"
33-
r2d2_sqlite = "0.20.0"
34-
rusqlite = { version = "0.27.0", features = ["chrono", "functions", "bundled"] }
35-
rand = "0.5"
34+
r2d2_sqlite = "0.21.0"
35+
rusqlite = { version = "0.28.0", features = ["chrono", "functions", "bundled"] }
36+
rand = "0.8"
3637
regex = "1.0"
3738
reqwest = { version = "0.11", features = ["blocking", "json"] }
3839
serde = "1.0"
@@ -45,29 +46,29 @@ tar = "0.4.36"
4546
tempfile = "3.0.0"
4647
tera = "0.11.7"
4748
toml = "0.4.6"
48-
url = "1.1"
49+
url = "2"
4950
walkdir = "2"
5051
warp = "0.1.9"
5152
log = "0.4.6"
52-
env_logger = "0.6.0"
53-
hmac = "0.7"
54-
sha-1 = "0.8"
53+
env_logger = "0.9.0"
54+
hmac = "0.12"
55+
sha-1 = "0.10"
5556
rust_team_data = { git = "https://github.com/rust-lang/team" }
5657
systemstat = "0.1.11"
5758
rustwide = { version = "0.14.0", features = ["unstable", "unstable-toolchain-ci"] }
5859
percent-encoding = "2.1.0"
59-
remove_dir_all = "0.5.2"
60+
remove_dir_all = "0.7"
6061
ctrlc = "3.1.3"
6162
prometheus = "0.7.0"
6263
cargo_metadata = "0.12.1"
6364
indexmap = "1.4.0"
6465
tokio = "1.18"
65-
aws-types = { version = "0.12", features = ["hardcoded-credentials"] }
66-
aws-smithy-http = "0.42"
67-
aws-sdk-s3 = "0.12"
66+
aws-types = { version = "0.46", features = ["hardcoded-credentials"] }
67+
aws-smithy-http = "0.46"
68+
aws-sdk-s3 = "0.16"
6869

6970
[dev-dependencies]
70-
assert_cmd = "1.0.2"
71+
assert_cmd = "2.0.4"
7172
difference = "2.0.0"
7273
num_cpus = "1.8.0"
7374
predicates = "1.0.0"

‎src/crates/lists.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use crate::db::{Database, QueryUtils};
55
use crate::experiments::CrateSelect;
66
use crate::prelude::*;
77
use chrono::Utc;
8-
use rand::{thread_rng, Rng};
8+
use rand::{seq::SliceRandom, thread_rng};
99
use std::collections::HashSet;
1010

1111
pub(crate) use crate::crates::sources::{
@@ -141,7 +141,7 @@ pub(crate) fn get_crates(
141141
crates.append(&mut GitHubList::get(db)?);
142142

143143
let mut rng = thread_rng();
144-
rng.shuffle(&mut crates);
144+
crates.shuffle(&mut rng);
145145
crates.truncate(*n as usize);
146146
}
147147
CrateSelect::Top(n) => {

‎src/crates/sources/registry.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,12 @@ impl List for RegistryList {
1515
let mut counts = HashMap::new();
1616

1717
fs::create_dir_all(&*WORK_DIR)?;
18-
let index = Index::new(WORK_DIR.join("crates.io-index"));
19-
index.retrieve_or_update().to_failure()?;
18+
let mut index = Index::with_path(
19+
WORK_DIR.join("crates.io-index"),
20+
"https://github.com/rust-lang/crates.io-index",
21+
)
22+
.to_failure()?;
23+
index.update().to_failure()?;
2024

2125
for krate in index.crates() {
2226
// The versions() method returns the list of published versions starting from the

‎src/db/migrations.rs

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
use crate::prelude::*;
2-
use rand::{self, distributions::Alphanumeric, Rng};
2+
use rand::{
3+
self,
4+
distributions::{Alphanumeric, DistString},
5+
};
36
use rusqlite::{Connection, Transaction};
47
use std::collections::HashSet;
58

@@ -137,10 +140,7 @@ fn migrations() -> Vec<(&'static str, MigrationKind)> {
137140

138141
let fn_name = format!(
139142
"crater_migration__{}",
140-
rand::thread_rng()
141-
.sample_iter(&Alphanumeric)
142-
.take(10)
143-
.collect::<String>()
143+
Alphanumeric.sample_string(&mut rand::thread_rng(), 10)
144144
);
145145
t.create_scalar_function(
146146
&fn_name,
@@ -332,10 +332,7 @@ fn migrations() -> Vec<(&'static str, MigrationKind)> {
332332

333333
let fn_name = format!(
334334
"crater_migration__{}",
335-
rand::thread_rng()
336-
.sample_iter(&Alphanumeric)
337-
.take(10)
338-
.collect::<String>()
335+
Alphanumeric.sample_string(&mut rand::thread_rng(), 10)
339336
);
340337
t.create_scalar_function(
341338
&fn_name,

‎src/report/archives.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,8 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
122122
} else {
123123
std::thread::sleep(std::time::Duration::from_secs(2));
124124
warn!(
125-
"retry ({}/{}) writing logs-archives/all.tar.gz ({} bytes)",
126-
i, RETRIES, len,
125+
"retry ({}/{}) writing logs-archives/all.tar.gz ({} bytes) (error: {:?})",
126+
i, RETRIES, len, e,
127127
);
128128
continue;
129129
}

‎src/report/mod.rs

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,11 @@ pub fn generate_report<DB: ReadResults>(
217217
crates: &[Crate],
218218
) -> Fallible<RawTestResults> {
219219
let mut crates = crates.to_vec();
220-
let index = Index::new(WORK_DIR.join("crates.io-index"));
220+
let index = Index::with_path(
221+
WORK_DIR.join("crates.io-index"),
222+
"https://github.com/rust-lang/crates.io-index",
223+
)
224+
.to_failure()?;
221225
//crate ids are unique so unstable sort is equivalent to stable sort but is generally faster
222226
crates.sort_unstable_by_key(|a| a.id());
223227
let res = crates
@@ -694,8 +698,12 @@ mod tests {
694698
};
695699
let gh = Crate::GitHub(repo);
696700

697-
let index = Index::new(WORK_DIR.join("crates.io-index"));
698-
index.retrieve_or_update().unwrap();
701+
let mut index = Index::with_path(
702+
WORK_DIR.join("crates.io-index"),
703+
"https://github.com/rust-lang/crates.io-index",
704+
)
705+
.unwrap();
706+
index.update().unwrap();
699707

700708
assert_eq!(
701709
get_crate_version_status(&index, &reg).unwrap().unwrap(),

‎src/report/s3.rs

Lines changed: 103 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,11 @@ impl FromStr for S3Prefix {
4646

4747
Ok(S3Prefix {
4848
bucket,
49-
prefix: parsed.path()[1..].into(),
49+
prefix: parsed
50+
.path()
51+
.get(1..)
52+
.map(PathBuf::from)
53+
.unwrap_or_default(),
5054
})
5155
}
5256
}
@@ -77,28 +81,105 @@ impl ReportWriter for S3Writer {
7781
mime: &Mime,
7882
encoding_type: EncodingType,
7983
) -> Fallible<()> {
80-
let mut request = self
81-
.client
82-
.put_object()
83-
.body(aws_smithy_http::byte_stream::ByteStream::from(s))
84-
.acl(aws_sdk_s3::model::ObjectCannedAcl::PublicRead)
85-
.key(format!(
86-
"{}/{}",
87-
self.prefix,
88-
path.as_ref().to_str().unwrap()
89-
))
90-
.content_type(mime.to_string())
91-
.bucket(self.bucket.clone());
92-
match encoding_type {
93-
EncodingType::Plain => {}
94-
EncodingType::Gzip => {
95-
request = request.content_encoding("gzip");
84+
// At least 50 MB, then use a multipart upload...
85+
if s.len() >= 50 * 1024 * 1024 {
86+
let mut request = self
87+
.client
88+
.create_multipart_upload()
89+
.acl(aws_sdk_s3::model::ObjectCannedAcl::PublicRead)
90+
.key(format!(
91+
"{}/{}",
92+
self.prefix,
93+
path.as_ref().to_str().unwrap()
94+
))
95+
.content_type(mime.to_string())
96+
.bucket(self.bucket.clone());
97+
match encoding_type {
98+
EncodingType::Plain => {}
99+
EncodingType::Gzip => {
100+
request = request.content_encoding("gzip");
101+
}
96102
}
97-
}
98-
match self.runtime.block_on(request.send()) {
99-
Ok(_) => Ok(()),
100-
Err(e) => {
101-
failure::bail!("Failed to upload to {:?}: {:?}", path.as_ref(), e);
103+
let upload = match self.runtime.block_on(request.send()) {
104+
Ok(u) => u,
105+
Err(e) => {
106+
failure::bail!("Failed to upload to {:?}: {:?}", path.as_ref(), e);
107+
}
108+
};
109+
110+
let chunk_size = 20 * 1024 * 1024;
111+
let bytes = bytes_1::Bytes::from(s);
112+
let mut part = 1;
113+
let mut start = 0;
114+
let mut parts = aws_sdk_s3::model::CompletedMultipartUpload::builder();
115+
while start < bytes.len() {
116+
let chunk = bytes.slice(start..std::cmp::min(start + chunk_size, bytes.len()));
117+
118+
let request = self
119+
.client
120+
.upload_part()
121+
.part_number(part)
122+
.body(chunk.into())
123+
.upload_id(upload.upload_id().unwrap())
124+
.key(upload.key().unwrap())
125+
.bucket(self.bucket.clone());
126+
match self.runtime.block_on(request.send()) {
127+
Ok(p) => {
128+
parts = parts.parts(
129+
aws_sdk_s3::model::CompletedPart::builder()
130+
.e_tag(p.e_tag.clone().unwrap())
131+
.part_number(part)
132+
.build(),
133+
)
134+
}
135+
Err(e) => {
136+
failure::bail!("Failed to upload to {:?}: {:?}", path.as_ref(), e);
137+
}
138+
};
139+
140+
start += chunk_size;
141+
part += 1;
142+
}
143+
144+
let request = self
145+
.client
146+
.complete_multipart_upload()
147+
.multipart_upload(parts.build())
148+
.upload_id(upload.upload_id().unwrap())
149+
.key(upload.key().unwrap())
150+
.bucket(self.bucket.clone());
151+
match self.runtime.block_on(request.send()) {
152+
Ok(_) => (),
153+
Err(e) => {
154+
failure::bail!("Failed to upload to {:?}: {:?}", path.as_ref(), e);
155+
}
156+
};
157+
158+
Ok(())
159+
} else {
160+
let mut request = self
161+
.client
162+
.put_object()
163+
.body(aws_smithy_http::byte_stream::ByteStream::from(s))
164+
.acl(aws_sdk_s3::model::ObjectCannedAcl::PublicRead)
165+
.key(format!(
166+
"{}/{}",
167+
self.prefix,
168+
path.as_ref().to_str().unwrap()
169+
))
170+
.content_type(mime.to_string())
171+
.bucket(self.bucket.clone());
172+
match encoding_type {
173+
EncodingType::Plain => {}
174+
EncodingType::Gzip => {
175+
request = request.content_encoding("gzip");
176+
}
177+
}
178+
match self.runtime.block_on(request.send()) {
179+
Ok(_) => Ok(()),
180+
Err(e) => {
181+
failure::bail!("Failed to upload to {:?}: {:?}", path.as_ref(), e);
182+
}
102183
}
103184
}
104185
}

‎src/server/routes/webhooks/mod.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,9 +185,9 @@ fn verify_signature(secret: &str, payload: &[u8], raw_signature: &str) -> bool {
185185
}
186186

187187
// Verify the HMAC signature
188-
let mut mac = HmacSha1::new_varkey(secret.as_bytes()).unwrap();
189-
mac.input(payload);
190-
mac.verify(&signature).is_ok()
188+
let mut mac = HmacSha1::new_from_slice(secret.as_bytes()).unwrap();
189+
mac.update(payload);
190+
mac.verify_slice(&signature).is_ok()
191191
}
192192

193193
fn receive_endpoint(

‎tests/minicrater/driver.rs

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::common::CommandCraterExt;
22
use assert_cmd::prelude::*;
33
use difference::Changeset;
4-
use rand::{self, distributions::Alphanumeric, Rng};
4+
use rand::distributions::{Alphanumeric, DistString};
55
use serde_json::{self, Value};
66
use std::env;
77
use std::path::{Path, PathBuf};
@@ -148,10 +148,7 @@ impl MinicraterRun {
148148
let ex_arg = format!(
149149
"--ex=minicrater-{}-{}",
150150
self.ex,
151-
rand::thread_rng()
152-
.sample_iter(&Alphanumeric)
153-
.take(10)
154-
.collect::<String>()
151+
Alphanumeric.sample_string(&mut rand::thread_rng(), 10)
155152
);
156153

157154
// Create local list in the temp work dir

0 commit comments

Comments
 (0)