Skip to content

Commit 0f7cf47

Browse files
UgnineSirdisUgnineSirdis
authored
Audit log json envelope (#11466)
1 parent ee51155 commit 0f7cf47

File tree

8 files changed

+359
-19
lines changed

8 files changed

+359
-19
lines changed

‎ydb/core/audit/audit_log_impl.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ void WriteLog(const TString& log, const TVector<THolder<TLogBackend>>& logBacken
7373
log.length()
7474
));
7575
} catch (const yexception& e) {
76-
LOG_W("WriteLog: unable to write audit log (error: " << e.what() << ")");
76+
LOG_E("WriteLog: unable to write audit log (error: " << e.what() << ")");
7777
}
7878
}
7979
}

‎ydb/core/log_backend/json_envelope.cpp

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#include "json_envelope.h"
2+
3+
#include <util/charset/utf8.h>
4+
5+
namespace NKikimr {
6+
7+
const TStringBuf PLACEHOLDER = "%message%";
8+
9+
void TJsonEnvelope::Parse() {
10+
ReadJsonTree(TemplateString, &Value, true);
11+
std::vector<TReplace::TPathComponent> path;
12+
Parse(Value, path);
13+
}
14+
15+
bool TJsonEnvelope::Parse(const NJson::TJsonValue& value, std::vector<TReplace::TPathComponent>& path) {
16+
Y_ASSERT(!Replace);
17+
18+
switch (value.GetType()) {
19+
case NJson::JSON_STRING: {
20+
if (auto replacePair = Parse(value.GetStringSafe())) {
21+
Replace.emplace(std::move(path), std::move(replacePair->first), std::move(replacePair->second));
22+
return true;
23+
}
24+
return false;
25+
}
26+
case NJson::JSON_ARRAY: {
27+
const auto& arr = value.GetArraySafe();
28+
path.emplace_back(size_t(0));
29+
for (size_t i = 0; i < arr.size(); ++i) {
30+
path.back() = i;
31+
if (Parse(arr[i], path)) {
32+
return true;
33+
}
34+
}
35+
path.pop_back();
36+
return false;
37+
}
38+
case NJson::JSON_MAP: {
39+
const auto& map = value.GetMapSafe();
40+
path.emplace_back(TString());
41+
for (const auto& [key, el] : map) {
42+
path.back() = key;
43+
if (Parse(el, path)) {
44+
return true;
45+
}
46+
}
47+
path.pop_back();
48+
return false;
49+
}
50+
default:
51+
return false;
52+
}
53+
}
54+
55+
std::optional<std::pair<TString, TString>> TJsonEnvelope::Parse(const TString& stringValue) {
56+
std::optional<std::pair<TString, TString>> result;
57+
size_t pos = stringValue.find(PLACEHOLDER);
58+
if (pos == TString::npos) {
59+
return result;
60+
}
61+
62+
TString prefix, suffix;
63+
if (pos != 0) {
64+
prefix = stringValue.substr(0, pos);
65+
}
66+
if (pos + PLACEHOLDER.size() < stringValue.size()) {
67+
suffix = stringValue.substr(pos + PLACEHOLDER.size());
68+
}
69+
result.emplace(std::move(prefix), std::move(suffix));
70+
return result;
71+
}
72+
73+
TString TJsonEnvelope::ApplyJsonEnvelope(const TStringBuf& message) const {
74+
if (!IsUtf(message)) {
75+
throw std::runtime_error("Attempt to write non utf-8 string");
76+
}
77+
78+
const NJson::TJsonValue* value = &Value;
79+
NJson::TJsonValue valueCopy;
80+
if (Replace) {
81+
valueCopy = Value;
82+
value = &valueCopy;
83+
Replace->Apply(&valueCopy, message);
84+
}
85+
86+
TStringStream ss;
87+
NJson::WriteJson(&ss, value, NJson::TJsonWriterConfig().SetValidateUtf8(true));
88+
ss << Endl;
89+
return ss.Str();
90+
}
91+
92+
void TJsonEnvelope::TReplace::Apply(NJson::TJsonValue* value, const TStringBuf& message) const {
93+
size_t currentEl = 0;
94+
while (currentEl < Path.size()) {
95+
std::visit(
96+
[&](const auto& child) {
97+
value = &(*value)[child];
98+
},
99+
Path[currentEl++]
100+
);
101+
}
102+
103+
Y_ASSERT(value && value->GetType() == NJson::JSON_STRING);
104+
TString result;
105+
result.reserve(Prefix.size() + Suffix.size() + message.size());
106+
if (Prefix) {
107+
result += Prefix;
108+
}
109+
result += message;
110+
if (Suffix) {
111+
result += Suffix;
112+
}
113+
*value = result;
114+
}
115+
116+
} // namespace NKikimr

‎ydb/core/log_backend/json_envelope.h

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#pragma once
2+
#include <library/cpp/json/json_reader.h>
3+
#include <library/cpp/json/json_writer.h>
4+
5+
#include <util/generic/string.h>
6+
7+
#include <optional>
8+
#include <variant>
9+
#include <vector>
10+
11+
namespace NKikimr {
12+
13+
class TJsonEnvelope {
14+
struct TReplace {
15+
using TPathComponent = std::variant<size_t, TString>; // field or index
16+
std::vector<TPathComponent> Path;
17+
// replace
18+
TString Prefix;
19+
TString Suffix;
20+
21+
TReplace(std::vector<TPathComponent> path, TString prefix, TString suffix)
22+
: Path(std::move(path))
23+
, Prefix(std::move(prefix))
24+
, Suffix(std::move(suffix))
25+
{}
26+
27+
void Apply(NJson::TJsonValue* value, const TStringBuf& message) const;
28+
};
29+
30+
public:
31+
explicit TJsonEnvelope(const TString& templateString)
32+
: TemplateString(templateString)
33+
{
34+
Parse(); // can throw
35+
}
36+
37+
TJsonEnvelope() = delete;
38+
TJsonEnvelope(const TJsonEnvelope&) = delete;
39+
TJsonEnvelope(TJsonEnvelope&&) = delete;
40+
41+
TString ApplyJsonEnvelope(const TStringBuf& message) const;
42+
43+
private:
44+
void Parse();
45+
bool Parse(const NJson::TJsonValue& value, std::vector<TReplace::TPathComponent>& path);
46+
std::optional<std::pair<TString, TString>> Parse(const TString& stringValue); // returns prefix/suffix pair for replace
47+
48+
private:
49+
TString TemplateString;
50+
NJson::TJsonValue Value;
51+
std::optional<TReplace> Replace;
52+
};
53+
54+
} // namespace NKikimr

‎ydb/core/log_backend/json_envelope_ut.cpp

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#include "json_envelope.h"
2+
3+
#include <library/cpp/json/json_reader.h>
4+
#include <library/cpp/json/json_writer.h>
5+
#include <library/cpp/testing/unittest/registar.h>
6+
7+
namespace NKikimr {
8+
9+
#define UNIT_ASSERT_JSONS_EQUAL(j1, j2) { \
10+
const TString js1 = (j1), js2 = (j2); \
11+
UNIT_ASSERT(!js1.empty()); \
12+
UNIT_ASSERT_C(js1.back() == '\n', js1); \
13+
NJson::TJsonValue jv1, jv2; \
14+
UNIT_ASSERT(ReadJsonTree(j1, &jv1)); \
15+
UNIT_ASSERT(ReadJsonTree(j2, &jv2)); \
16+
const TString jsn1 = NJson::WriteJson(&jv1, true, true); \
17+
const TString jsn2 = NJson::WriteJson(&jv2, true, true); \
18+
UNIT_ASSERT_VALUES_EQUAL(jsn1, jsn2); \
19+
}
20+
21+
Y_UNIT_TEST_SUITE(JsonEnvelopeTest) {
22+
Y_UNIT_TEST(Simple) {
23+
TJsonEnvelope env1(R"json({
24+
"a": "b",
25+
"m": "abc%message%def"
26+
})json");
27+
28+
UNIT_ASSERT_JSONS_EQUAL(env1.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"abcmsgdef"})json");
29+
UNIT_ASSERT_JSONS_EQUAL(env1.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"abcxyzdef"})json");
30+
31+
32+
TJsonEnvelope env2(R"json({
33+
"a": "b",
34+
"m": "%message%def"
35+
})json");
36+
37+
UNIT_ASSERT_JSONS_EQUAL(env2.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"msgdef"})json");
38+
UNIT_ASSERT_JSONS_EQUAL(env2.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"xyzdef"})json");
39+
40+
41+
TJsonEnvelope env3(R"json({
42+
"a": "b",
43+
"m": "abc%message%"
44+
})json");
45+
46+
UNIT_ASSERT_JSONS_EQUAL(env3.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"abcmsg"})json");
47+
UNIT_ASSERT_JSONS_EQUAL(env3.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"abcxyz"})json");
48+
}
49+
50+
Y_UNIT_TEST(NoReplace) {
51+
TJsonEnvelope env(R"json({
52+
"a": "b",
53+
"x": "%y%",
54+
"subfield": {
55+
"s": "% message %",
56+
"t": "%Message%",
57+
"x": 42,
58+
"a": [
59+
42
60+
]
61+
}
62+
})json");
63+
64+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"b","x":"%y%","subfield":{"s":"% message %","t":"%Message%","x":42,"a":[42]}})json");
65+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("xyz"), R"json({"a":"b","x":"%y%","subfield":{"s":"% message %","t":"%Message%","x":42,"a":[42]}})json");
66+
}
67+
68+
Y_UNIT_TEST(ArrayItem) {
69+
TJsonEnvelope env(R"json({
70+
"a": "b",
71+
"subfield": {
72+
"a": [
73+
42,
74+
"%message%",
75+
53
76+
]
77+
}
78+
})json");
79+
80+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"b","subfield":{"a":[42,"msg",53]}})json");
81+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("xyz"), R"json({"a":"b","subfield":{"a":[42,"xyz",53]}})json");
82+
}
83+
84+
Y_UNIT_TEST(Escape) {
85+
TJsonEnvelope env(R"json({
86+
"a": "%message%"
87+
})json");
88+
89+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"msg"})json");
90+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("\"\n\""), R"json({"a":"\"\n\""})json");
91+
}
92+
93+
Y_UNIT_TEST(BinaryData) {
94+
TJsonEnvelope env(R"json({
95+
"a": "%message%"
96+
})json");
97+
98+
const ui64 binaryData = 0xABCDEFFF87654321;
99+
const TStringBuf data(reinterpret_cast<const char*>(&binaryData), sizeof(binaryData));
100+
UNIT_ASSERT_EXCEPTION(env.ApplyJsonEnvelope(data), std::exception);
101+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("text"), R"json({"a":"text"})json");
102+
}
103+
}
104+
105+
} // namespace NKikimr

0 commit comments

Comments
 (0)