Skip to content

Commit 1d7f6a2

Browse files
mergify[bot]fanatid
mergify[bot]
and
fanatid
authored
Rpc: filters performance improvement (solana-labs#20185) (solana-labs#20862)
* Add Base58,Base64,Bytes to MemcmpEncodedBytes * Rpc: decode memcmp before filtering accounts * Add deprecated attribute * Add Memcmp::bytes * Fix clippy for deprecated * Another clippy fix * merge RpcFilterError::DataTooLarge * add deprecation for Base58DataTooLarge * change filter data size limit * strict data size len for base58 * add magic numbers * fix tests (cherry picked from commit e9a427b) Co-authored-by: Kirill Fomichev <fanatid@ya.ru>
1 parent a483a7d commit 1d7f6a2

File tree

4 files changed

+147
-53
lines changed

4 files changed

+147
-53
lines changed

‎cli/src/cluster_query.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1721,15 +1721,15 @@ pub fn process_show_stakes(
17211721
// Filter by `StakeState::Stake(_, _)`
17221722
rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
17231723
offset: 0,
1724-
bytes: rpc_filter::MemcmpEncodedBytes::Binary(
1724+
bytes: rpc_filter::MemcmpEncodedBytes::Base58(
17251725
bs58::encode([2, 0, 0, 0]).into_string(),
17261726
),
17271727
encoding: Some(rpc_filter::MemcmpEncoding::Binary),
17281728
}),
17291729
// Filter by `Delegation::voter_pubkey`, which begins at byte offset 124
17301730
rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
17311731
offset: 124,
1732-
bytes: rpc_filter::MemcmpEncodedBytes::Binary(
1732+
bytes: rpc_filter::MemcmpEncodedBytes::Base58(
17331733
vote_account_pubkeys[0].to_string(),
17341734
),
17351735
encoding: Some(rpc_filter::MemcmpEncoding::Binary),

‎cli/src/program.rs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1146,18 +1146,18 @@ fn get_buffers(
11461146
) -> Result<CliUpgradeableBuffers, Box<dyn std::error::Error>> {
11471147
let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
11481148
offset: 0,
1149-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 0, 0, 0]).into_string()),
1149+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 0, 0, 0]).into_string()),
11501150
encoding: None,
11511151
})];
11521152
if let Some(authority_pubkey) = authority_pubkey {
11531153
filters.push(RpcFilterType::Memcmp(Memcmp {
11541154
offset: ACCOUNT_TYPE_SIZE,
1155-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
1155+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
11561156
encoding: None,
11571157
}));
11581158
filters.push(RpcFilterType::Memcmp(Memcmp {
11591159
offset: ACCOUNT_TYPE_SIZE + OPTION_SIZE,
1160-
bytes: MemcmpEncodedBytes::Binary(
1160+
bytes: MemcmpEncodedBytes::Base58(
11611161
bs58::encode(authority_pubkey.as_ref()).into_string(),
11621162
),
11631163
encoding: None,
@@ -1199,18 +1199,18 @@ fn get_programs(
11991199
) -> Result<CliUpgradeablePrograms, Box<dyn std::error::Error>> {
12001200
let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
12011201
offset: 0,
1202-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 0, 0, 0]).into_string()),
1202+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 0, 0, 0]).into_string()),
12031203
encoding: None,
12041204
})];
12051205
if let Some(authority_pubkey) = authority_pubkey {
12061206
filters.push(RpcFilterType::Memcmp(Memcmp {
12071207
offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE,
1208-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
1208+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
12091209
encoding: None,
12101210
}));
12111211
filters.push(RpcFilterType::Memcmp(Memcmp {
12121212
offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE + OPTION_SIZE,
1213-
bytes: MemcmpEncodedBytes::Binary(
1213+
bytes: MemcmpEncodedBytes::Base58(
12141214
bs58::encode(authority_pubkey.as_ref()).into_string(),
12151215
),
12161216
encoding: None,
@@ -1234,7 +1234,7 @@ fn get_programs(
12341234
bytes.extend_from_slice(programdata_address.as_ref());
12351235
let filters = vec![RpcFilterType::Memcmp(Memcmp {
12361236
offset: 0,
1237-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(bytes).into_string()),
1237+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(bytes).into_string()),
12381238
encoding: None,
12391239
})];
12401240

‎client/src/rpc_filter.rs

Lines changed: 104 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
use thiserror::Error;
1+
#![allow(deprecated)]
2+
use {std::borrow::Cow, thiserror::Error};
3+
4+
const MAX_DATA_SIZE: usize = 128;
5+
const MAX_DATA_BASE58_SIZE: usize = 175;
6+
const MAX_DATA_BASE64_SIZE: usize = 172;
27

38
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
49
#[serde(rename_all = "camelCase")]
@@ -15,15 +20,50 @@ impl RpcFilterType {
1520
let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary);
1621
match encoding {
1722
MemcmpEncoding::Binary => {
18-
let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes;
19-
20-
if bytes.len() > 128 {
21-
Err(RpcFilterError::Base58DataTooLarge)
22-
} else {
23-
bs58::decode(&bytes)
24-
.into_vec()
25-
.map(|_| ())
26-
.map_err(|e| e.into())
23+
use MemcmpEncodedBytes::*;
24+
match &compare.bytes {
25+
Binary(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
26+
Err(RpcFilterError::Base58DataTooLarge)
27+
}
28+
Base58(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
29+
Err(RpcFilterError::DataTooLarge)
30+
}
31+
Base64(bytes) if bytes.len() > MAX_DATA_BASE64_SIZE => {
32+
Err(RpcFilterError::DataTooLarge)
33+
}
34+
Bytes(bytes) if bytes.len() > MAX_DATA_SIZE => {
35+
Err(RpcFilterError::DataTooLarge)
36+
}
37+
_ => Ok(()),
38+
}?;
39+
match &compare.bytes {
40+
Binary(bytes) => {
41+
let bytes = bs58::decode(&bytes)
42+
.into_vec()
43+
.map_err(RpcFilterError::DecodeError)?;
44+
if bytes.len() > MAX_DATA_SIZE {
45+
Err(RpcFilterError::Base58DataTooLarge)
46+
} else {
47+
Ok(())
48+
}
49+
}
50+
Base58(bytes) => {
51+
let bytes = bs58::decode(&bytes).into_vec()?;
52+
if bytes.len() > MAX_DATA_SIZE {
53+
Err(RpcFilterError::DataTooLarge)
54+
} else {
55+
Ok(())
56+
}
57+
}
58+
Base64(bytes) => {
59+
let bytes = base64::decode(&bytes)?;
60+
if bytes.len() > MAX_DATA_SIZE {
61+
Err(RpcFilterError::DataTooLarge)
62+
} else {
63+
Ok(())
64+
}
65+
}
66+
Bytes(_) => Ok(()),
2767
}
2868
}
2969
}
@@ -34,10 +74,24 @@ impl RpcFilterType {
3474

3575
#[derive(Error, PartialEq, Debug)]
3676
pub enum RpcFilterError {
37-
#[error("bs58 decode error")]
38-
DecodeError(#[from] bs58::decode::Error),
77+
#[error("encoded binary data should be less than 129 bytes")]
78+
DataTooLarge,
79+
#[deprecated(
80+
since = "1.9.0",
81+
note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
82+
)]
3983
#[error("encoded binary (base 58) data should be less than 129 bytes")]
4084
Base58DataTooLarge,
85+
#[deprecated(
86+
since = "1.9.0",
87+
note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
88+
)]
89+
#[error("bs58 decode error")]
90+
DecodeError(bs58::decode::Error),
91+
#[error("base58 decode error")]
92+
Base58DecodeError(#[from] bs58::decode::Error),
93+
#[error("base64 decode error")]
94+
Base64DecodeError(#[from] base64::DecodeError),
4195
}
4296

4397
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -49,7 +103,14 @@ pub enum MemcmpEncoding {
49103
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
50104
#[serde(rename_all = "camelCase", untagged)]
51105
pub enum MemcmpEncodedBytes {
106+
#[deprecated(
107+
since = "1.9.0",
108+
note = "Please use MemcmpEncodedBytes::Base58 instead"
109+
)]
52110
Binary(String),
111+
Base58(String),
112+
Base64(String),
113+
Bytes(Vec<u8>),
53114
}
54115

55116
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -63,14 +124,18 @@ pub struct Memcmp {
63124
}
64125

65126
impl Memcmp {
66-
pub fn bytes_match(&self, data: &[u8]) -> bool {
127+
pub fn bytes(&self) -> Option<Cow<Vec<u8>>> {
128+
use MemcmpEncodedBytes::*;
67129
match &self.bytes {
68-
MemcmpEncodedBytes::Binary(bytes) => {
69-
let bytes = bs58::decode(bytes).into_vec();
70-
if bytes.is_err() {
71-
return false;
72-
}
73-
let bytes = bytes.unwrap();
130+
Binary(bytes) | Base58(bytes) => bs58::decode(bytes).into_vec().ok().map(Cow::Owned),
131+
Base64(bytes) => base64::decode(bytes).ok().map(Cow::Owned),
132+
Bytes(bytes) => Some(Cow::Borrowed(bytes)),
133+
}
134+
}
135+
136+
pub fn bytes_match(&self, data: &[u8]) -> bool {
137+
match self.bytes() {
138+
Some(bytes) => {
74139
if self.offset > data.len() {
75140
return false;
76141
}
@@ -79,6 +144,7 @@ impl Memcmp {
79144
}
80145
data[self.offset..self.offset + bytes.len()] == bytes[..]
81146
}
147+
None => false,
82148
}
83149
}
84150
}
@@ -87,62 +153,71 @@ impl Memcmp {
87153
mod tests {
88154
use super::*;
89155

156+
#[test]
157+
fn test_worst_case_encoded_tx_goldens() {
158+
let ff_data = vec![0xffu8; MAX_DATA_SIZE];
159+
let data58 = bs58::encode(&ff_data).into_string();
160+
assert_eq!(data58.len(), MAX_DATA_BASE58_SIZE);
161+
let data64 = base64::encode(&ff_data);
162+
assert_eq!(data64.len(), MAX_DATA_BASE64_SIZE);
163+
}
164+
90165
#[test]
91166
fn test_bytes_match() {
92167
let data = vec![1, 2, 3, 4, 5];
93168

94169
// Exact match of data succeeds
95170
assert!(Memcmp {
96171
offset: 0,
97-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
172+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
98173
encoding: None,
99174
}
100175
.bytes_match(&data));
101176

102177
// Partial match of data succeeds
103178
assert!(Memcmp {
104179
offset: 0,
105-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()),
180+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2]).into_string()),
106181
encoding: None,
107182
}
108183
.bytes_match(&data));
109184

110185
// Offset partial match of data succeeds
111186
assert!(Memcmp {
112187
offset: 2,
113-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()),
188+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4]).into_string()),
114189
encoding: None,
115190
}
116191
.bytes_match(&data));
117192

118193
// Incorrect partial match of data fails
119194
assert!(!Memcmp {
120195
offset: 0,
121-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()),
196+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![2]).into_string()),
122197
encoding: None,
123198
}
124199
.bytes_match(&data));
125200

126201
// Bytes overrun data fails
127202
assert!(!Memcmp {
128203
offset: 2,
129-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()),
204+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4, 5, 6]).into_string()),
130205
encoding: None,
131206
}
132207
.bytes_match(&data));
133208

134209
// Offset outside data fails
135210
assert!(!Memcmp {
136211
offset: 6,
137-
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()),
212+
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![5]).into_string()),
138213
encoding: None,
139214
}
140215
.bytes_match(&data));
141216

142217
// Invalid base-58 fails
143218
assert!(!Memcmp {
144219
offset: 0,
145-
bytes: MemcmpEncodedBytes::Binary("III".to_string()),
220+
bytes: MemcmpEncodedBytes::Base58("III".to_string()),
146221
encoding: None,
147222
}
148223
.bytes_match(&data));
@@ -157,7 +232,7 @@ mod tests {
157232
assert_eq!(
158233
RpcFilterType::Memcmp(Memcmp {
159234
offset: 0,
160-
bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
235+
bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
161236
encoding: None,
162237
})
163238
.verify(),
@@ -172,11 +247,11 @@ mod tests {
172247
assert_eq!(
173248
RpcFilterType::Memcmp(Memcmp {
174249
offset: 0,
175-
bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
250+
bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
176251
encoding: None,
177252
})
178253
.verify(),
179-
Err(RpcFilterError::Base58DataTooLarge)
254+
Err(RpcFilterError::DataTooLarge)
180255
);
181256
}
182257
}

0 commit comments

Comments
 (0)