Skip to content

Commit 8dbf241

Browse files
adameatadameat
authored
Merge viewer fixes Apr 29th (#17839)
2 parents fce90cb + 47bba61 commit 8dbf241

32 files changed

+1280
-479
lines changed

‎ydb/core/driver_lib/run/run.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,7 @@ void TKikimrRunner::InitializeMonitoring(const TKikimrRunConfig& runConfig, bool
475475
if (securityConfig.MonitoringAllowedSIDsSize() > 0) {
476476
monConfig.AllowedSIDs.assign(securityConfig.GetMonitoringAllowedSIDs().begin(), securityConfig.GetMonitoringAllowedSIDs().end());
477477
}
478+
monConfig.AllowOrigin = appConfig.GetMonitoringConfig().GetAllowOrigin();
478479

479480
if (ModuleFactories && ModuleFactories->MonitoringFactory) {
480481
Monitoring = ModuleFactories->MonitoringFactory(std::move(monConfig), appConfig);

‎ydb/core/mind/hive/hive_impl.cpp

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2092,9 +2092,18 @@ void THive::Handle(TEvHive::TEvRequestHiveNodeStats::TPtr& ev) {
20922092
}
20932093
}
20942094
} else {
2095+
std::optional<TSubDomainKey> filterObjectDomain;
2096+
if (request.HasFilterTabletsByObjectDomain()) {
2097+
filterObjectDomain = TSubDomainKey(request.GetFilterTabletsByObjectDomain());
2098+
}
20952099
for (const auto& [state, set] : node.Tablets) {
20962100
std::vector<ui32> tabletTypeToCount;
20972101
for (const TTabletInfo* tablet : set) {
2102+
if (filterObjectDomain) {
2103+
if (tablet->NodeFilter.ObjectDomain != *filterObjectDomain) {
2104+
continue;
2105+
}
2106+
}
20982107
TTabletTypes::EType type = tablet->GetTabletType();
20992108
if (static_cast<size_t>(type) >= tabletTypeToCount.size()) {
21002109
tabletTypeToCount.resize(type + 1);
@@ -3553,7 +3562,7 @@ void THive::Handle(TEvPrivate::TEvUpdateFollowers::TPtr&) {
35533562

35543563
void THive::MakeScaleRecommendation() {
35553564
BLOG_D("[MSR] Started");
3556-
3565+
35573566
if (AreWeRootHive()) {
35583567
return;
35593568
}

‎ydb/core/mon/mon.cpp

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include <library/cpp/lwtrace/mon/mon_lwtrace.h>
1717
#include <ydb/library/actors/core/probes.h>
1818
#include <ydb/core/base/monitoring_provider.h>
19+
#include <ydb/core/util/wildcard.h>
1920

2021
#include <library/cpp/monlib/service/pages/version_mon_page.h>
2122
#include <library/cpp/monlib/service/pages/mon_page.h>
@@ -403,7 +404,19 @@ class THttpMonLegacyActorRequest : public TActorBootstrapped<THttpMonLegacyActor
403404
type = "application/json";
404405
}
405406
NHttp::THeaders headers(request->Headers);
406-
TString origin = TString(headers["Origin"]);
407+
TString allowOrigin = AppData()->Mon->GetConfig().AllowOrigin;
408+
TString requestOrigin = TString(headers["Origin"]);
409+
TString origin;
410+
if (allowOrigin) {
411+
if (IsMatchesWildcards(requestOrigin, allowOrigin)) {
412+
origin = requestOrigin;
413+
} else {
414+
Send(Event->Sender, new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(request->CreateResponseBadRequest("Invalid CORS origin")));
415+
return PassAway();
416+
}
417+
} else if (requestOrigin) {
418+
origin = requestOrigin;
419+
}
407420
if (origin.empty()) {
408421
origin = "*";
409422
}

‎ydb/core/mon/mon.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class TMon {
4040
TString Certificate;
4141
ui32 MaxRequestsPerSecond = 0;
4242
TDuration InactivityTimeout = TDuration::Minutes(2);
43+
TString AllowOrigin;
4344
};
4445

4546
TMon(TConfig config);
@@ -86,6 +87,10 @@ class TMon {
8687
});
8788
}
8889

90+
const TConfig& GetConfig() const {
91+
return Config;
92+
}
93+
8994
protected:
9095
TConfig Config;
9196
TIntrusivePtr<NMonitoring::TIndexMonPage> IndexMonPage;

‎ydb/core/protos/hive.proto

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,7 @@ message TEvRequestHiveNodeStats {
323323
// The next 2 fields should always be used together
324324
optional uint64 FilterTabletsByPathId = 7;
325325
optional uint64 FilterTabletsBySchemeShardId = 8;
326+
optional NKikimrSubDomains.TDomainKey FilterTabletsByObjectDomain = 9;
326327
}
327328

328329
message THiveNodeStats {
@@ -616,7 +617,7 @@ message TScaleRecommenderPolicies {
616617
message TScaleRecommenderPolicy {
617618
message TTargetTrackingPolicy {
618619
oneof Target {
619-
uint32 AverageCpuUtilizationPercent = 1;
620+
uint32 AverageCpuUtilizationPercent = 1;
620621
}
621622
}
622623

‎ydb/core/protos/node_whiteboard.proto

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,16 @@ enum EConfigState {
281281
Outdated = 1;
282282
}
283283

284+
message TSystemThreadInfo {
285+
optional string Name = 1;
286+
optional uint32 Threads = 2;
287+
optional double SystemUsage = 3;
288+
optional double UserUsage = 4;
289+
optional uint32 MinorPageFaults = 5;
290+
optional uint32 MajorPageFaults = 6;
291+
map<string, uint32> States = 7;
292+
}
293+
284294
message TSystemStateInfo {
285295
message TPoolStats {
286296
optional string Name = 1;
@@ -346,6 +356,8 @@ message TSystemStateInfo {
346356
optional uint32 CoresTotal = 40;
347357
optional float NetworkUtilization = 41;
348358
optional uint64 NetworkWriteThroughput = 42;
359+
optional uint32 RealNumberOfCpus = 43; // number of cpus without cgroups limitations
360+
repeated TSystemThreadInfo Threads = 44;
349361
}
350362

351363
message TEvSystemStateRequest {

‎ydb/core/tablet/node_whiteboard.cpp

Lines changed: 60 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <ydb/core/node_whiteboard/node_whiteboard.h>
1010
#include <ydb/core/base/nameservice.h>
1111
#include <ydb/core/base/counters.h>
12+
#include <ydb/core/util/cpuinfo.h>
1213
#include <ydb/core/util/tuples.h>
1314

1415
#include <util/string/split.h>
@@ -46,6 +47,7 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
4647
SystemStateInfo.SetNodeName(nodeName);
4748
}
4849
SystemStateInfo.SetNumberOfCpus(NSystemInfo::NumberOfCpus());
50+
SystemStateInfo.SetRealNumberOfCpus(NKikimr::RealNumberOfCpus());
4951
auto version = GetProgramRevision();
5052
if (!version.empty()) {
5153
SystemStateInfo.SetVersion(version);
@@ -56,8 +58,13 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
5658
SystemStateInfo.SetStartTime(ctx.Now().MilliSeconds());
5759
ctx.Send(ctx.SelfID, new TEvPrivate::TEvUpdateRuntimeStats());
5860

59-
auto group = NKikimr::GetServiceCounters(NKikimr::AppData()->Counters, "utils")
60-
->GetSubgroup("subsystem", "whiteboard");
61+
auto utils = NKikimr::GetServiceCounters(NKikimr::AppData()->Counters, "utils");
62+
UserTime = utils->GetCounter("Process/UserTime", true);
63+
SysTime = utils->GetCounter("Process/SystemTime", true);
64+
MinorPageFaults = utils->GetCounter("Process/MinorPageFaults", true);
65+
MajorPageFaults = utils->GetCounter("Process/MajorPageFaults", true);
66+
NumThreads = utils->GetCounter("Process/NumThreads", false);
67+
auto group = utils->GetSubgroup("subsystem", "whiteboard");
6168
MaxClockSkewWithPeerUsCounter = group->GetCounter("MaxClockSkewWithPeerUs");
6269
MaxClockSkewPeerIdCounter = group->GetCounter("MaxClockSkewPeerId");
6370

@@ -78,8 +85,19 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
7885
NKikimrWhiteboard::TSystemStateInfo SystemStateInfo;
7986
THolder<NTracing::ITraceCollection> TabletIntrospectionData;
8087

81-
::NMonitoring::TDynamicCounters::TCounterPtr MaxClockSkewWithPeerUsCounter;
82-
::NMonitoring::TDynamicCounters::TCounterPtr MaxClockSkewPeerIdCounter;
88+
NMonitoring::TDynamicCounters::TCounterPtr MaxClockSkewWithPeerUsCounter;
89+
NMonitoring::TDynamicCounters::TCounterPtr MaxClockSkewPeerIdCounter;
90+
NMonitoring::TDynamicCounters::TCounterPtr UserTime;
91+
ui64 SavedUserTime = 0;
92+
NMonitoring::TDynamicCounters::TCounterPtr SysTime;
93+
ui64 SavedSysTime = 0;
94+
NMonitoring::TDynamicCounters::TCounterPtr MinorPageFaults;
95+
ui64 SavedMinorPageFaults = 0;
96+
NMonitoring::TDynamicCounters::TCounterPtr MajorPageFaults;
97+
ui64 SavedMajorPageFaults = 0;
98+
NMonitoring::TDynamicCounters::TCounterPtr NumThreads;
99+
100+
TSystemThreadsMonitor ThreadsMonitor;
83101

84102
template <typename PropertyType>
85103
static ui64 GetDifference(PropertyType a, PropertyType b) {
@@ -721,6 +739,9 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
721739
auto& endpoint = *SystemStateInfo.AddEndpoints();
722740
endpoint.SetName(ev->Get()->Name);
723741
endpoint.SetAddress(ev->Get()->Address);
742+
std::sort(SystemStateInfo.MutableEndpoints()->begin(), SystemStateInfo.MutableEndpoints()->end(), [](const auto& a, const auto& b) {
743+
return a.GetName() < b.GetName();
744+
});
724745
SystemStateInfo.SetChangeTime(ctx.Now().MilliSeconds());
725746
}
726747

@@ -838,14 +859,25 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
838859
}
839860
}
840861

862+
static std::unordered_set<TTabletId> BuildIndex(const ::google::protobuf::RepeatedField<::NProtoBuf::uint64>& array) {
863+
std::unordered_set<TTabletId> result;
864+
result.reserve(array.size());
865+
for (auto id : array) {
866+
result.insert(id);
867+
}
868+
return result;
869+
}
870+
841871
void Handle(TEvWhiteboard::TEvTabletStateRequest::TPtr &ev, const TActorContext &ctx) {
842872
auto now = TMonotonic::Now();
843873
const auto& request = ev->Get()->Record;
844874
auto matchesFilter = [
845875
changedSince = request.has_changedsince() ? request.changedsince() : 0,
876+
filterTabletId = BuildIndex(request.filtertabletid()),
846877
filterTenantId = request.has_filtertenantid() ? NKikimr::TSubDomainKey(request.filtertenantid()) : NKikimr::TSubDomainKey()
847878
](const NKikimrWhiteboard::TTabletStateInfo& tabletStateInfo) {
848879
return tabletStateInfo.changetime() >= changedSince
880+
&& (filterTabletId.empty() || filterTabletId.count(tabletStateInfo.tabletid()))
849881
&& (!filterTenantId || filterTenantId == NKikimr::TSubDomainKey(tabletStateInfo.tenantid()));
850882
};
851883
std::unique_ptr<TEvWhiteboard::TEvTabletStateResponse> response = std::make_unique<TEvWhiteboard::TEvTabletStateResponse>();
@@ -868,22 +900,10 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
868900
}
869901
} else {
870902
if (request.groupby().empty()) {
871-
if (request.filtertabletid_size() == 0) {
872-
for (const auto& pr : TabletStateInfo) {
873-
if (matchesFilter(pr.second)) {
874-
NKikimrWhiteboard::TTabletStateInfo& tabletStateInfo = *record.add_tabletstateinfo();
875-
Copy(tabletStateInfo, pr.second, request);
876-
}
877-
}
878-
} else {
879-
for (auto tabletId : request.filtertabletid()) {
880-
auto it = TabletStateInfo.find({tabletId, 0});
881-
if (it != TabletStateInfo.end()) {
882-
if (matchesFilter(it->second)) {
883-
NKikimrWhiteboard::TTabletStateInfo& tabletStateInfo = *record.add_tabletstateinfo();
884-
Copy(tabletStateInfo, it->second, request);
885-
}
886-
}
903+
for (const auto& pr : TabletStateInfo) {
904+
if (matchesFilter(pr.second)) {
905+
NKikimrWhiteboard::TTabletStateInfo& tabletStateInfo = *record.add_tabletstateinfo();
906+
Copy(tabletStateInfo, pr.second, request);
887907
}
888908
}
889909
} else if (request.groupby() == "Type,State" || request.groupby() == "NodeId,Type,State") { // the only supported group-by for now
@@ -1097,15 +1117,18 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
10971117
}
10981118

10991119
void Handle(TEvPrivate::TEvUpdateRuntimeStats::TPtr &, const TActorContext &ctx) {
1100-
static constexpr TDuration UPDATE_PERIOD = TDuration::Seconds(15);
1120+
static constexpr int UPDATE_PERIOD_SECONDS = 15;
1121+
static constexpr TDuration UPDATE_PERIOD = TDuration::Seconds(UPDATE_PERIOD_SECONDS);
1122+
auto now = TActivationContext::Now();
1123+
11011124
{
11021125
NKikimrWhiteboard::TSystemStateInfo systemStatsUpdate;
11031126
TVector<double> loadAverage = GetLoadAverage();
11041127
for (double d : loadAverage) {
11051128
systemStatsUpdate.AddLoadAverage(d);
11061129
}
11071130
if (CheckedMerge(SystemStateInfo, systemStatsUpdate)) {
1108-
SystemStateInfo.SetChangeTime(ctx.Now().MilliSeconds());
1131+
SystemStateInfo.SetChangeTime(now.MilliSeconds());
11091132
}
11101133
}
11111134

@@ -1122,12 +1145,24 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
11221145
SystemStateInfo.SetNetworkUtilization(MaxNetworkUtilization);
11231146
MaxNetworkUtilization = 0;
11241147
}
1125-
11261148
{
1127-
SystemStateInfo.SetNetworkWriteThroughput(SumNetworkWriteThroughput / UPDATE_PERIOD.Seconds());
1149+
SystemStateInfo.SetNetworkWriteThroughput(SumNetworkWriteThroughput / UPDATE_PERIOD_SECONDS);
11281150
SumNetworkWriteThroughput = 0;
11291151
}
1130-
1152+
auto threadPools = ThreadsMonitor.GetThreadPools(now);
1153+
SystemStateInfo.ClearThreads();
1154+
for (const auto& threadPool : threadPools) {
1155+
auto* threadInfo = SystemStateInfo.AddThreads();
1156+
threadInfo->SetName(threadPool.Name);
1157+
threadInfo->SetThreads(threadPool.Threads);
1158+
threadInfo->SetSystemUsage(threadPool.SystemUsage);
1159+
threadInfo->SetUserUsage(threadPool.UserUsage);
1160+
threadInfo->SetMajorPageFaults(threadPool.MajorPageFaults);
1161+
threadInfo->SetMinorPageFaults(threadPool.MinorPageFaults);
1162+
for (const auto& state : threadPool.States) {
1163+
threadInfo->MutableStates()->emplace(state.first, state.second);
1164+
}
1165+
}
11311166
UpdateSystemState(ctx);
11321167
ctx.Schedule(UPDATE_PERIOD, new TEvPrivate::TEvUpdateRuntimeStats());
11331168
}

‎ydb/core/tx/schemeshard/schemeshard_path_describer.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,10 @@ void TPathDescriber::FillChildDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElem
180180
descr->SetCreateStep(ui64(pathEl->StepCreated));
181181
}
182182

183-
descr->SetChildrenExist(pathEl->GetAliveChildren() > 0);
183+
if (pathEl->PathType != NKikimrSchemeOp::EPathTypeSubDomain
184+
&& pathEl->PathType != NKikimrSchemeOp::EPathTypeExtSubDomain) {
185+
descr->SetChildrenExist(pathEl->GetAliveChildren() > 0);
186+
}
184187

185188
if (pathEl->PathType == NKikimrSchemeOp::EPathTypePersQueueGroup) {
186189
auto it = Self->Topics.FindPtr(pathEl->PathId);

0 commit comments

Comments
 (0)