Skip to content

Commit a27cce8

Browse files
romangeromange
authored
feat: add support for meta memcache commands (#4362)
This is a stripped down version of supporting the memcache meta requests. a. Not all meta flags are supported, but TTL, flags, arithmetics are supported. b. does not include reply support. c. does not include new semantics that are not part of the older, ascii protocol. The parser interface has not changed significantly, and the meta commands are emulated using the old, high level commands like ADD,REPLACE, INCR etc. See https://raw.githubusercontent.com/memcached/memcached/refs/heads/master/doc/protocol.txt for more details regarding the meta commands spec. Signed-off-by: Roman Gershman <roman@dragonflydb.io>
1 parent 6946820 commit a27cce8

File tree

7 files changed

+291
-22
lines changed

7 files changed

+291
-22
lines changed

‎src/facade/dragonfly_connection.cc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1137,8 +1137,14 @@ auto Connection::ParseMemcache() -> ParserStatus {
11371137

11381138
do {
11391139
string_view str = ToSV(io_buf_.InputBuffer());
1140+
1141+
if (str.empty()) {
1142+
return OK;
1143+
}
1144+
11401145
result = memcache_parser_->Parse(str, &consumed, &cmd);
11411146

1147+
DVLOG(2) << "mc_result " << result << " consumed: " << consumed << " type " << cmd.type;
11421148
if (result != MemcacheParser::OK) {
11431149
io_buf_.ConsumeInput(consumed);
11441150
break;

‎src/facade/memcache_parser.cc

Lines changed: 193 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@
66
#include <absl/container/flat_hash_map.h>
77
#include <absl/container/inlined_vector.h>
88
#include <absl/strings/ascii.h>
9+
#include <absl/strings/escaping.h>
910
#include <absl/strings/numbers.h>
1011
#include <absl/strings/str_split.h>
1112
#include <absl/types/span.h>
1213

1314
#include "base/logging.h"
1415
#include "base/stl_util.h"
16+
#include "facade/facade_types.h"
1517

1618
namespace facade {
1719
using namespace std;
@@ -29,16 +31,37 @@ MP::CmdType From(string_view token) {
2931
{"quit", MP::QUIT}, {"version", MP::VERSION},
3032
};
3133

32-
auto it = cmd_map.find(token);
33-
if (it == cmd_map.end())
34+
if (token.size() == 2) {
35+
// META_COMMANDS
36+
if (token[0] != 'm')
37+
return MP::INVALID;
38+
switch (token[1]) {
39+
case 's':
40+
return MP::META_SET;
41+
case 'g':
42+
return MP::META_GET;
43+
case 'd':
44+
return MP::META_DEL;
45+
case 'a':
46+
return MP::META_ARITHM;
47+
case 'n':
48+
return MP::META_NOOP;
49+
case 'e':
50+
return MP::META_DEBUG;
51+
}
3452
return MP::INVALID;
53+
}
3554

36-
return it->second;
55+
if (token.size() > 2) {
56+
auto it = cmd_map.find(token);
57+
if (it == cmd_map.end())
58+
return MP::INVALID;
59+
return it->second;
60+
}
61+
return MP::INVALID;
3762
}
3863

39-
using TokensView = absl::Span<std::string_view>;
40-
41-
MP::Result ParseStore(TokensView tokens, MP::Command* res) {
64+
MP::Result ParseStore(ArgSlice tokens, MP::Command* res) {
4265
const size_t num_tokens = tokens.size();
4366
unsigned opt_pos = 3;
4467
if (res->type == MP::CAS) {
@@ -70,7 +93,7 @@ MP::Result ParseStore(TokensView tokens, MP::Command* res) {
7093
return MP::OK;
7194
}
7295

73-
MP::Result ParseValueless(TokensView tokens, MP::Command* res) {
96+
MP::Result ParseValueless(ArgSlice tokens, MP::Command* res) {
7497
const size_t num_tokens = tokens.size();
7598
size_t key_pos = 0;
7699
if (res->type == MP::GAT || res->type == MP::GATS) {
@@ -116,13 +139,159 @@ MP::Result ParseValueless(TokensView tokens, MP::Command* res) {
116139
return MP::OK;
117140
}
118141

142+
bool ParseMetaMode(char m, MP::Command* res) {
143+
if (res->type == MP::SET) {
144+
switch (m) {
145+
case 'E':
146+
res->type = MP::ADD;
147+
break;
148+
case 'A':
149+
res->type = MP::APPEND;
150+
break;
151+
case 'R':
152+
res->type = MP::REPLACE;
153+
break;
154+
case 'P':
155+
res->type = MP::PREPEND;
156+
break;
157+
case 'S':
158+
break;
159+
default:
160+
return false;
161+
}
162+
return true;
163+
}
164+
165+
if (res->type == MP::INCR) {
166+
switch (m) {
167+
case 'I':
168+
case '+':
169+
break;
170+
case 'D':
171+
case '-':
172+
res->type = MP::DECR;
173+
break;
174+
default:
175+
return false;
176+
}
177+
return true;
178+
}
179+
return false;
180+
}
181+
182+
// See https://raw.githubusercontent.com/memcached/memcached/refs/heads/master/doc/protocol.txt
183+
MP::Result ParseMeta(ArgSlice tokens, MP::Command* res) {
184+
DCHECK(!tokens.empty());
185+
186+
if (res->type == MP::META_DEBUG) {
187+
LOG(ERROR) << "meta debug not yet implemented";
188+
return MP::PARSE_ERROR;
189+
}
190+
191+
if (tokens[0].size() > 250)
192+
return MP::PARSE_ERROR;
193+
194+
res->meta = true;
195+
res->key = tokens[0];
196+
res->bytes_len = 0;
197+
res->flags = 0;
198+
res->expire_ts = 0;
199+
200+
tokens.remove_prefix(1);
201+
202+
// We emulate the behavior by returning the high level commands.
203+
// TODO: we should reverse the interface in the future, so that a high level command
204+
// will be represented in MemcacheParser::Command by a meta command with flags.
205+
// high level commands should not be part of the interface in the future.
206+
switch (res->type) {
207+
case MP::META_GET:
208+
res->type = MP::GET;
209+
break;
210+
case MP::META_DEL:
211+
res->type = MP::DELETE;
212+
break;
213+
case MP::META_SET:
214+
if (tokens.empty()) {
215+
return MP::PARSE_ERROR;
216+
}
217+
if (!absl::SimpleAtoi(tokens[0], &res->bytes_len))
218+
return MP::BAD_INT;
219+
220+
res->type = MP::SET;
221+
tokens.remove_prefix(1);
222+
break;
223+
case MP::META_ARITHM:
224+
res->type = MP::INCR;
225+
res->delta = 1;
226+
break;
227+
default:
228+
return MP::PARSE_ERROR;
229+
}
230+
231+
for (size_t i = 0; i < tokens.size(); ++i) {
232+
string_view token = tokens[i];
233+
234+
switch (token[0]) {
235+
case 'T':
236+
if (!absl::SimpleAtoi(token.substr(1), &res->expire_ts))
237+
return MP::BAD_INT;
238+
break;
239+
case 'b':
240+
if (token.size() != 1)
241+
return MP::PARSE_ERROR;
242+
if (!absl::Base64Unescape(res->key, &res->blob))
243+
return MP::PARSE_ERROR;
244+
res->key = res->blob;
245+
res->base64 = true;
246+
break;
247+
case 'F':
248+
if (!absl::SimpleAtoi(token.substr(1), &res->flags))
249+
return MP::BAD_INT;
250+
break;
251+
case 'M':
252+
if (token.size() != 2 || !ParseMetaMode(token[1], res))
253+
return MP::PARSE_ERROR;
254+
break;
255+
case 'D':
256+
if (!absl::SimpleAtoi(token.substr(1), &res->delta))
257+
return MP::BAD_INT;
258+
break;
259+
case 'q':
260+
res->no_reply = true;
261+
break;
262+
case 'f':
263+
res->return_flags = true;
264+
break;
265+
case 'v':
266+
res->return_value = true;
267+
break;
268+
case 't':
269+
res->return_ttl = true;
270+
break;
271+
case 'l':
272+
res->return_access_time = true;
273+
break;
274+
case 'h':
275+
res->return_hit = true;
276+
break;
277+
default:
278+
LOG(WARNING) << "unknown meta flag: " << token; // not yet implemented
279+
return MP::PARSE_ERROR;
280+
}
281+
}
282+
283+
return MP::OK;
284+
}
285+
119286
} // namespace
120287

121288
auto MP::Parse(string_view str, uint32_t* consumed, Command* cmd) -> Result {
122289
cmd->no_reply = false; // re-initialize
123290
auto pos = str.find("\r\n");
124291
*consumed = 0;
125292
if (pos == string_view::npos) {
293+
// We need more data to parse the command. For get/gets commands this line can be very long.
294+
// we limit maxmimum buffer capacity in the higher levels using max_client_iobuf_len.
126295
return INPUT_PENDING;
127296
}
128297

@@ -131,42 +300,47 @@ auto MP::Parse(string_view str, uint32_t* consumed, Command* cmd) -> Result {
131300
}
132301
*consumed = pos + 2;
133302

134-
std::string_view tokens_expression = str.substr(0, pos);
303+
string_view tokens_expression = str.substr(0, pos);
135304

136305
// cas <key> <flags> <exptime> <bytes> <cas unique> [noreply]\r\n
137306
// get <key>*\r\n
138-
absl::InlinedVector<std::string_view, 32> tokens =
307+
// ms <key> <datalen> <flags>*\r\n
308+
absl::InlinedVector<string_view, 32> tokens =
139309
absl::StrSplit(tokens_expression, ' ', absl::SkipWhitespace());
140310

141-
const size_t num_tokens = tokens.size();
142-
143-
if (num_tokens == 0)
311+
if (tokens.empty())
144312
return PARSE_ERROR;
145313

146314
cmd->type = From(tokens[0]);
147315
if (cmd->type == INVALID) {
148316
return UNKNOWN_CMD;
149317
}
150318

151-
if (cmd->type <= CAS) { // Store command
152-
if (num_tokens < 5 || tokens[1].size() > 250) { // key length limit
319+
ArgSlice tokens_view{tokens};
320+
tokens_view.remove_prefix(1);
321+
322+
if (cmd->type <= CAS) { // Store command
323+
if (tokens_view.size() < 4 || tokens[0].size() > 250) { // key length limit
153324
return MP::PARSE_ERROR;
154325
}
155326

156-
cmd->key = string_view{tokens[1].data(), tokens[1].size()};
327+
cmd->key = tokens_view[0];
157328

158-
TokensView tokens_view{tokens.begin() + 2, num_tokens - 2};
329+
tokens_view.remove_prefix(1);
159330
return ParseStore(tokens_view, cmd);
160331
}
161332

162-
if (num_tokens == 1) {
163-
if (base::_in(cmd->type, {MP::STATS, MP::FLUSHALL, MP::QUIT, MP::VERSION})) {
333+
if (cmd->type >= META_SET) {
334+
return tokens_view.empty() ? MP::PARSE_ERROR : ParseMeta(tokens_view, cmd);
335+
}
336+
337+
if (tokens_view.empty()) {
338+
if (base::_in(cmd->type, {MP::STATS, MP::FLUSHALL, MP::QUIT, MP::VERSION, MP::META_NOOP})) {
164339
return MP::OK;
165340
}
166341
return MP::PARSE_ERROR;
167342
}
168343

169-
TokensView tokens_view{tokens.begin() + 1, num_tokens - 1};
170344
return ParseValueless(tokens_view, cmd);
171345
};
172346

‎src/facade/memcache_parser.h

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#pragma once
66

77
#include <cstdint>
8+
#include <string>
89
#include <string_view>
910
#include <vector>
1011

@@ -39,6 +40,14 @@ class MemcacheParser {
3940
INCR = 32,
4041
DECR = 33,
4142
FLUSHALL = 34,
43+
44+
// META_COMMANDS
45+
META_NOOP = 50,
46+
META_SET = 51,
47+
META_DEL = 52,
48+
META_ARITHM = 53,
49+
META_GET = 54,
50+
META_DEBUG = 55,
4251
};
4352

4453
// According to https://github.com/memcached/memcached/wiki/Commands#standard-protocol
@@ -56,15 +65,26 @@ class MemcacheParser {
5665
0; // relative (expire_ts <= month) or unix time (expire_ts > month) in seconds
5766
uint32_t bytes_len = 0;
5867
uint32_t flags = 0;
59-
bool no_reply = false;
68+
bool no_reply = false; // q
69+
bool meta = false;
70+
71+
// meta flags
72+
bool base64 = false; // b
73+
bool return_flags = false; // f
74+
bool return_value = false; // v
75+
bool return_ttl = false; // t
76+
bool return_access_time = false; // l
77+
bool return_hit = false; // h
78+
// Used internally by meta parsing.
79+
std::string blob;
6080
};
6181

6282
enum Result {
6383
OK,
6484
INPUT_PENDING,
6585
UNKNOWN_CMD,
6686
BAD_INT,
67-
PARSE_ERROR,
87+
PARSE_ERROR, // request parse error, but can continue parsing within the same connection.
6888
BAD_DELTA,
6989
};
7090

0 commit comments

Comments
 (0)