Skip to content

Commit 6966c4e

Browse files
azygisazygis
authored
Merge pull request #376 from hangfire-postgres/features/333-aggregated-stats
Read aggregated counter table too for hourly stats
2 parents e69f90d + 93324ec commit 6966c4e

File tree

5 files changed

+159
-29
lines changed

5 files changed

+159
-29
lines changed

‎src/Hangfire.PostgreSql/CountersAggregator.cs

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -82,28 +82,26 @@ private string GetAggregationQuery()
8282
{
8383
string schemaName = _storage.Options.SchemaName;
8484
return
85-
$@"BEGIN;
85+
$"""
86+
BEGIN;
8687
87-
INSERT INTO ""{schemaName}"".""aggregatedcounter"" (""key"", ""value"", ""expireat"")
88-
SELECT
89-
""key"",
90-
SUM(""value""),
91-
MAX(""expireat"")
92-
FROM ""{schemaName}"".""counter""
93-
GROUP BY
94-
""key""
95-
ON CONFLICT(""key"") DO
96-
UPDATE
97-
SET
98-
""value"" = ""aggregatedcounter"".""value"" + EXCLUDED.""value"",
99-
""expireat"" = EXCLUDED.""expireat"";
88+
INSERT INTO "{schemaName}"."aggregatedcounter" ("key", "value", "expireat")
89+
SELECT
90+
"key",
91+
SUM("value"),
92+
MAX("expireat")
93+
FROM "{schemaName}"."counter"
94+
GROUP BY "key"
95+
ON CONFLICT("key") DO UPDATE
96+
SET "value" = "aggregatedcounter"."value" + EXCLUDED."value", "expireat" = EXCLUDED."expireat";
97+
98+
DELETE FROM "{schemaName}"."counter"
99+
WHERE "key" IN (
100+
SELECT "key" FROM "{schemaName}"."aggregatedcounter"
101+
);
100102
101-
DELETE FROM ""{schemaName}"".""counter""
102-
WHERE
103-
""key"" IN (SELECT ""key"" FROM ""{schemaName}"".""aggregatedcounter"" );
104-
105-
COMMIT;
106-
";
103+
COMMIT;
104+
""";
107105
}
108106
}
109107
}

‎src/Hangfire.PostgreSql/PostgreSqlMonitoringApi.cs

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -379,12 +379,25 @@ private Dictionary<DateTime, long> GetTimelineStats(string type)
379379

380380
private Dictionary<DateTime, long> GetTimelineStats(IDictionary<string, DateTime> keyMaps)
381381
{
382-
string query = $@"
383-
SELECT ""key"", COUNT(""value"") AS ""count""
384-
FROM ""{_storage.Options.SchemaName}"".""counter""
385-
WHERE ""key"" = ANY (@Keys)
386-
GROUP BY ""key"";
387-
";
382+
string query =
383+
$"""
384+
WITH "aggregated_counters" AS (
385+
SELECT "key", "value"
386+
FROM "{_storage.Options.SchemaName}"."aggregatedcounter"
387+
WHERE "key" = ANY(@Keys)
388+
), "regular_counters" AS (
389+
SELECT "key", "value"
390+
FROM "{_storage.Options.SchemaName}"."counter"
391+
WHERE "key" = ANY(@Keys)
392+
), "all_counters" AS (
393+
SELECT * FROM "aggregated_counters"
394+
UNION ALL
395+
SELECT * FROM "regular_counters"
396+
)
397+
SELECT "key", COALESCE(SUM("value"), 0) AS "count"
398+
FROM "all_counters"
399+
GROUP BY "key"
400+
""";
388401

389402
Dictionary<string, long> valuesMap = UseConnection(connection => connection.Query<(string Key, long Count)>(query,
390403
new { Keys = keyMaps.Keys.ToList() })
@@ -400,10 +413,10 @@ private Dictionary<DateTime, long> GetTimelineStats(IDictionary<string, DateTime
400413
}
401414

402415
Dictionary<DateTime, long> result = new();
403-
for (int i = 0; i < keyMaps.Count; i++)
416+
foreach (KeyValuePair<string, DateTime> keyMap in keyMaps)
404417
{
405-
long value = valuesMap[keyMaps.ElementAt(i).Key];
406-
result.Add(keyMaps.ElementAt(i).Value, value);
418+
long value = valuesMap[keyMap.Key];
419+
result.Add(keyMap.Value, value);
407420
}
408421

409422
return result;

‎tests/Hangfire.PostgreSql.Tests/CountersAggregatorFacts.cs

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
using System;
2+
using System.Threading;
3+
using Dapper;
4+
using Hangfire.PostgreSql.Tests.Utils;
5+
using Npgsql;
6+
using Xunit;
7+
8+
namespace Hangfire.PostgreSql.Tests;
9+
10+
public class CountersAggregatorFacts : IClassFixture<PostgreSqlStorageFixture>
11+
{
12+
private static readonly string _schemaName = ConnectionUtils.GetSchemaName();
13+
14+
private readonly CancellationToken _token;
15+
private readonly PostgreSqlStorageFixture _fixture;
16+
17+
public CountersAggregatorFacts(PostgreSqlStorageFixture fixture)
18+
{
19+
CancellationTokenSource cts = new();
20+
_token = cts.Token;
21+
_fixture = fixture;
22+
_fixture.SetupOptions(o => o.CountersAggregateInterval = TimeSpan.FromMinutes(5));
23+
}
24+
25+
[Fact]
26+
[CleanDatabase]
27+
public void Execute_AggregatesCounters()
28+
{
29+
UseConnection((connection, manager) => {
30+
CreateEntry(1);
31+
CreateEntry(5);
32+
CreateEntry(15);
33+
CreateEntry(5, "key2");
34+
CreateEntry(10, "key2");
35+
36+
manager.Execute(_token);
37+
38+
Assert.Equal(21, GetAggregatedCounters(connection));
39+
Assert.Equal(15, GetAggregatedCounters(connection, "key2"));
40+
Assert.Null(GetRegularCounters(connection));
41+
Assert.Null(GetRegularCounters(connection, "key2"));
42+
return;
43+
44+
void CreateEntry(long value, string key = "key")
45+
{
46+
CreateCounterEntry(connection, value, key);
47+
}
48+
});
49+
}
50+
51+
private void UseConnection(Action<NpgsqlConnection, CountersAggregator> action)
52+
{
53+
PostgreSqlStorage storage = _fixture.SafeInit();
54+
CountersAggregator aggregator = new(storage, TimeSpan.Zero);
55+
action(storage.CreateAndOpenConnection(), aggregator);
56+
}
57+
58+
private static void CreateCounterEntry(NpgsqlConnection connection, long? value, string key = "key")
59+
{
60+
value ??= 1;
61+
string insertSql =
62+
$"""
63+
INSERT INTO "{_schemaName}"."counter"("key", "value", "expireat")
64+
VALUES (@Key, @Value, null)
65+
""";
66+
67+
connection.Execute(insertSql, new { Key = key, Value = value });
68+
}
69+
70+
private static long GetAggregatedCounters(NpgsqlConnection connection, string key = "key")
71+
{
72+
return connection.QuerySingle<long>(
73+
$"""
74+
SELECT "value"
75+
FROM {_schemaName}."aggregatedcounter"
76+
WHERE "key" = @Key
77+
""", new { Key = key });
78+
}
79+
80+
private static long? GetRegularCounters(NpgsqlConnection connection, string key = "key")
81+
{
82+
return connection.QuerySingle<long?>(
83+
$"""
84+
SELECT SUM("value")
85+
FROM {_schemaName}."counter"
86+
WHERE "key" = @Key
87+
""", new { Key = key });
88+
}
89+
}

‎tests/Hangfire.PostgreSql.Tests/PostgreSqlMonitoringApiFacts.cs

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Globalization;
4+
using System.Linq;
45
using Dapper;
56
using Hangfire.Common;
67
using Hangfire.PostgreSql.Tests.Utils;
@@ -58,6 +59,34 @@ public void GetJobs_MixedCasing_ReturnsJob()
5859
});
5960
}
6061

62+
[Fact]
63+
[CleanDatabase]
64+
public void HourlySucceededJobs_ReturnsAggregatedStats()
65+
{
66+
DateTime now = DateTime.UtcNow;
67+
string schemaName = ConnectionUtils.GetSchemaName();
68+
string key = $"stats:succeeded:{now.ToString("yyyy-MM-dd-HH", CultureInfo.InvariantCulture)}";
69+
string arrangeSql =
70+
$"""
71+
BEGIN;
72+
INSERT INTO "{schemaName}"."counter"("key", "value")
73+
VALUES (@Key, 5);
74+
INSERT INTO "{schemaName}"."aggregatedcounter"("key", "value")
75+
VALUES (@Key, 7);
76+
COMMIT;
77+
""";
78+
UseConnection(connection => {
79+
connection.Execute(arrangeSql, new { Key = key });
80+
81+
IMonitoringApi monitoringApi = _fixture.Storage.GetMonitoringApi();
82+
IDictionary<DateTime, long> stats = monitoringApi.HourlySucceededJobs();
83+
Assert.Equal(24, stats.Count);
84+
85+
long actualCounter = Assert.Single(stats.Where(x => x.Key.Hour == now.Hour).Select(x => x.Value));
86+
Assert.Equal(12, actualCounter);
87+
});
88+
}
89+
6190
private void UseConnection(Action<NpgsqlConnection> action)
6291
{
6392
PostgreSqlStorage storage = _fixture.SafeInit();

‎tests/Hangfire.PostgreSql.Tests/Scripts/Clean.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
SET search_path = 'hangfire';
22

3+
DELETE FROM hangfire."aggregatedcounter";
34
DELETE FROM hangfire."counter";
45
DELETE FROM hangfire."hash";
56
DELETE FROM hangfire."job";

0 commit comments

Comments
 (0)