Skip to content

Commit af4b0f3

Browse files
wolffcmMongoDB Bot
wolffcm
authored and
MongoDB Bot
committed
SERVER-73641 Only generate timeseries _id comparisons on mongod (#29521)
GitOrigin-RevId: 8d053d1bc303b17d7e2c1598a1073d079c837446
1 parent af67844 commit af4b0f3

14 files changed

+1243
-247
lines changed

‎etc/backports_required_for_multiversion_tests.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,10 @@ last-continuous:
569569
ticket: SERVER-95670
570570
- test_file: jstests/sharding/query/owning_shard_expression.js
571571
ticket: SERVER-95670
572+
- test_file: jstests/sharding/timeseries_query_extended_range.js
573+
ticket: SERVER-73641
574+
- test_file: jstests/core/timeseries/timeseries_index_partial.js
575+
ticket: SERVER-73641
572576
suites: null
573577
last-lts:
574578
all:
@@ -1190,4 +1194,8 @@ last-lts:
11901194
ticket: SERVER-95670
11911195
- test_file: jstests/sharding/query/owning_shard_expression.js
11921196
ticket: SERVER-95670
1197+
- test_file: jstests/sharding/timeseries_query_extended_range.js
1198+
ticket: SERVER-73641
1199+
- test_file: jstests/core/timeseries/timeseries_index_partial.js
1200+
ticket: SERVER-73641
11931201
suites: null

‎jstests/core/timeseries/timeseries_delete_extended_range.js

Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
/**
2+
* This test exercises deletes on time series collections with and without extended range
3+
* data, and verifies results and that predicates are generated correctly. Before 8.0, we don't
4+
* push down predicates on the time field at all, and also don't generate any predicates on the _id
5+
* field for buckets, even if there are predicates on the time field.
6+
*
7+
* @tags: [
8+
* # We need a timeseries collection.
9+
* requires_timeseries,
10+
* requires_non_retryable_writes,
11+
* # Arbitrary timeseries delete support is not present before 7.0
12+
* requires_fcv_70
13+
* ]
14+
*/
15+
16+
(function() {
17+
load("jstests/libs/analyze_plan.js");
18+
19+
const timeFieldName = "time";
20+
const metaFieldName = "tag";
21+
22+
let testDB = null;
23+
24+
function getTestDB() {
25+
if (!testDB) {
26+
testDB = db.getSiblingDB(jsTestName());
27+
assert.commandWorked(testDB.dropDatabase());
28+
}
29+
return testDB;
30+
}
31+
32+
function prepareCollection({dbToUse, collName, initialDocList}) {
33+
if (!dbToUse) {
34+
dbToUse = getTestDB();
35+
}
36+
const coll = dbToUse.getCollection(collName);
37+
coll.drop();
38+
assert.commandWorked(dbToUse.createCollection(coll.getName(), {
39+
timeseries: {
40+
timeField: timeFieldName,
41+
metaField: metaFieldName,
42+
}
43+
}));
44+
assert.commandWorked(coll.insert(initialDocList));
45+
46+
return coll;
47+
}
48+
49+
function verifyResultDocs(coll, initialDocList, expectedResultDocs, nDeleted) {
50+
let resultDocs = coll.find().toArray();
51+
assert.eq(resultDocs.length, initialDocList.length - nDeleted, tojson(resultDocs));
52+
53+
assert.eq(expectedResultDocs.length, resultDocs.length, tojson(resultDocs));
54+
assert.sameMembers(expectedResultDocs, resultDocs, tojson(resultDocs));
55+
}
56+
57+
function verifyExplain({
58+
explain,
59+
residualFilter,
60+
nBucketsUnpacked,
61+
}) {
62+
jsTestLog(`Explain: ${tojson(explain)}`);
63+
let foundStage = getPlanStage(explain.queryPlanner.winningPlan, "TS_MODIFY");
64+
assert.neq(
65+
null, foundStage, `The root TS_MODIFY stage not found in the plan: ${tojson(explain)}`);
66+
67+
assert.eq("deleteMany", foundStage.opType, `TS_MODIFY opType is wrong: ${tojson(foundStage)}`);
68+
assert.eq({"control.closed": {$not: {$eq: true}}},
69+
foundStage.bucketFilter,
70+
`TS_MODIFY bucketFilter is wrong: ${tojson(foundStage)}`);
71+
assert.eq(residualFilter,
72+
foundStage.residualFilter,
73+
`TS_MODIFY residualFilter is wrong: ${tojson(foundStage)}`);
74+
75+
const execStages = getExecutionStages(explain);
76+
assert.eq("TS_MODIFY", execStages[0].stage, `The root stage is wrong: ${tojson(execStages)}`);
77+
let tsModifyStage = execStages[0];
78+
assert.eq(
79+
"TS_MODIFY", tsModifyStage.stage, `Can't find TS_MODIFY stage: ${tojson(execStages)}`);
80+
81+
assert.eq(nBucketsUnpacked,
82+
tsModifyStage.nBucketsUnpacked,
83+
`Got wrong nBucketsUnpacked ${tojson(tsModifyStage)}`);
84+
}
85+
86+
function getCallerName() {
87+
return `${new Error().stack.split('\n')[2].split('@')[0]}`;
88+
}
89+
90+
function makeDeleteCommand(coll, filter) {
91+
return {delete: coll.getName(), deletes: [{q: filter, limit: 0}]};
92+
}
93+
94+
/**
95+
* Verifies that a delete returns the expected result(s) 'res'.
96+
*
97+
* - initialDocList: The initial documents in the collection.
98+
* - cmd.filter: The filter for the findAndModify command.
99+
* - res.expectedResultDocs: The expected documents in the collection after the delete.
100+
* - res.nDeleted: The expected number of documents deleted.
101+
* - res.residualFilter: The expected residual filter of the TS_MODIFY stage.
102+
* - res.nBucketsUnpacked: The expected number of buckets unpacked by the TS_MODIFY stage.
103+
*/
104+
function testDelete({
105+
initialDocList,
106+
cmd: {filter},
107+
res: {
108+
expectedResultDocs,
109+
nDeleted,
110+
residualFilter,
111+
nBucketsUnpacked,
112+
}
113+
}) {
114+
const callerName = getCallerName();
115+
jsTestLog(`Running ${callerName}(${tojson(arguments[0])})`);
116+
117+
const coll = prepareCollection({
118+
collName: callerName,
119+
initialDocList: initialDocList,
120+
});
121+
122+
const deleteCmd = makeDeleteCommand(coll, filter);
123+
jsTestLog(`Running delete: ${tojson(deleteCmd)}`);
124+
125+
const explainRes =
126+
assert.commandWorked(coll.runCommand({explain: deleteCmd, verbosity: "executionStats"}));
127+
verifyExplain({
128+
explain: explainRes,
129+
residualFilter: residualFilter,
130+
nBucketsUnpacked: nBucketsUnpacked,
131+
});
132+
133+
const res = assert.commandWorked(testDB.runCommand(deleteCmd));
134+
jsTestLog(`delete result: ${tojson(res)}`);
135+
assert.eq(nDeleted, res.n, tojson(res));
136+
verifyResultDocs(coll, initialDocList, expectedResultDocs, nDeleted);
137+
}
138+
139+
// A set of documents that contain extended range data before and after the epoch.
140+
const extendedRangeDocs = [
141+
{
142+
[timeFieldName]: ISODate("1968-01-01T00:00:00Z"),
143+
[metaFieldName]: 1,
144+
_id: 0,
145+
a: 10,
146+
},
147+
{
148+
[timeFieldName]: ISODate("1971-01-01T00:00:00Z"),
149+
[metaFieldName]: 1,
150+
_id: 1,
151+
a: 10,
152+
},
153+
{
154+
[timeFieldName]: ISODate("2035-01-01T00:00:00Z"),
155+
[metaFieldName]: 1,
156+
_id: 2,
157+
a: 10,
158+
},
159+
{
160+
[timeFieldName]: ISODate("2040-01-01T00:00:00Z"),
161+
[metaFieldName]: 1,
162+
_id: 3,
163+
a: 10,
164+
},
165+
];
166+
167+
// A set of documents that does not contain extended range data.
168+
const normalDocs = [
169+
{
170+
[timeFieldName]: ISODate("1972-01-01T00:00:00Z"),
171+
[metaFieldName]: 1,
172+
_id: 0,
173+
a: 10,
174+
},
175+
{
176+
[timeFieldName]: ISODate("1975-01-01T00:00:00Z"),
177+
[metaFieldName]: 1,
178+
_id: 1,
179+
a: 10,
180+
},
181+
{
182+
[timeFieldName]: ISODate("2035-01-01T00:00:00Z"),
183+
[metaFieldName]: 1,
184+
_id: 2,
185+
a: 10,
186+
},
187+
{
188+
[timeFieldName]: ISODate("2038-01-01T00:00:00Z"),
189+
[metaFieldName]: 1,
190+
_id: 3,
191+
a: 10,
192+
},
193+
];
194+
195+
(function testDelete_extendedRange() {
196+
testDelete({
197+
initialDocList: extendedRangeDocs,
198+
cmd: {
199+
filter: {[timeFieldName]: {$gt: ISODate("1972-01-01T00:00:00Z")}},
200+
},
201+
res: {
202+
expectedResultDocs: [
203+
{
204+
[timeFieldName]: ISODate("1968-01-01T00:00:00Z"),
205+
[metaFieldName]: 1,
206+
_id: 0,
207+
a: 10,
208+
},
209+
{
210+
[timeFieldName]: ISODate("1971-01-01T00:00:00Z"),
211+
[metaFieldName]: 1,
212+
_id: 1,
213+
a: 10,
214+
},
215+
],
216+
residualFilter: {"time": {"$gt": ISODate("1972-01-01T00:00:00Z")}},
217+
nBucketsUnpacked: 4,
218+
nDeleted: 2
219+
},
220+
});
221+
})();
222+
223+
(function testDelete_normal() {
224+
testDelete({
225+
initialDocList: normalDocs,
226+
cmd: {
227+
filter: {[timeFieldName]: {$gt: ISODate("1972-01-01T00:00:00Z")}},
228+
},
229+
res: {
230+
expectedResultDocs: [
231+
{
232+
[timeFieldName]: ISODate("1972-01-01T00:00:00Z"),
233+
[metaFieldName]: 1,
234+
_id: 0,
235+
a: 10,
236+
},
237+
],
238+
residualFilter: {"time": {"$gt": ISODate("1972-01-01T00:00:00Z")}},
239+
nBucketsUnpacked: 4,
240+
nDeleted: 3
241+
},
242+
});
243+
})();
244+
245+
(function testDelete_normalExtendedPredicateAll() {
246+
testDelete({
247+
initialDocList: normalDocs,
248+
cmd: {
249+
filter: {[timeFieldName]: {$gt: ISODate("1950-01-01T00:00:00Z")}},
250+
},
251+
res: {
252+
expectedResultDocs: [],
253+
// The bucket filter is trivially true (empty predicate).
254+
// All the documents in the collection are after 1950.
255+
residualFilter: {"time": {"$gt": ISODate("1950-01-01T00:00:00Z")}},
256+
nBucketsUnpacked: 4,
257+
nDeleted: 4
258+
},
259+
});
260+
})();
261+
262+
(function testDelete_normalExtendedPredicateNone() {
263+
testDelete({
264+
initialDocList: normalDocs,
265+
cmd: {
266+
filter: {[timeFieldName]: {$lt: ISODate("1950-01-01T00:00:00Z")}},
267+
},
268+
res: {
269+
expectedResultDocs: normalDocs,
270+
// None of the documents in the collection can be before 1950.
271+
residualFilter: {"time": {"$lt": ISODate("1950-01-01T00:00:00Z")}},
272+
nBucketsUnpacked: 4,
273+
nDeleted: 0
274+
},
275+
});
276+
})();
277+
})();

‎jstests/core/timeseries/timeseries_index_partial.js

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -220,9 +220,6 @@ assert.sameMembers(buckets.getIndexes(), extraBucketIndexes.concat([
220220
{"control.max.time": {"$_internalExprGt": ISODate("2000-01-01T00:00:00Z")}},
221221
// We also have a bound on the min time, derived from bucketMaxSpanSeconds.
222222
{"control.min.time": {"$_internalExprGt": ISODate("1999-12-31T23:00:00Z")}},
223-
// The min time is also encoded in the _id, so we have a bound on that as
224-
// well.
225-
{"_id": {"$gt": ObjectId("386d3570ffffffffffffffff")}},
226223
]
227224
},
228225
// $gt on a non-time field can only bound the control.max for that field.
@@ -537,16 +534,16 @@ assert.sameMembers(buckets.getIndexes(), extraBucketIndexes.concat([
537534
// {a: {$in: [/abc/]}} is equivalent to {a: /abc/} which executes the regex, and is not allowed.
538535
checkPredicateDisallowed({a: {$in: [/abc/]}});
539536

540-
// Predicates on time are pushed down, and also some extra predicates are inferred.
541-
// These inferred predicates don't necessarily result in fewer buckets being indexed:
542-
// we're just following the same rules for createIndex that we use when optimizing a query.
537+
// Predicates on time are pushed down, being converted into predicates on the control field in
538+
// the process. These generated predicates don't necessarily result in fewer buckets being
539+
// indexed: we're just following the same rules for createIndex that we use when optimizing a
540+
// query.
543541
checkPredicateOK({
544542
input: {[timeField]: {$lt: ISODate('2020-01-01T00:00:00Z')}},
545543
output: {
546544
$and: [
547545
{"control.min.time": {"$_internalExprLt": ISODate("2020-01-01T00:00:00Z")}},
548546
{"control.max.time": {"$_internalExprLt": ISODate("2020-01-01T01:00:00Z")}},
549-
{"_id": {"$lt": ObjectId("5e0be1000000000000000000")}},
550547
]
551548
}
552549
});
@@ -556,7 +553,6 @@ assert.sameMembers(buckets.getIndexes(), extraBucketIndexes.concat([
556553
$and: [
557554
{"control.max.time": {"$_internalExprGt": ISODate("2020-01-01T00:00:00Z")}},
558555
{"control.min.time": {"$_internalExprGt": ISODate("2019-12-31T23:00:00Z")}},
559-
{"_id": {"$gt": ObjectId("5e0bd2f0ffffffffffffffff")}},
560556
]
561557
}
562558
});
@@ -566,7 +562,6 @@ assert.sameMembers(buckets.getIndexes(), extraBucketIndexes.concat([
566562
$and: [
567563
{"control.min.time": {"$_internalExprLte": ISODate("2020-01-01T00:00:00Z")}},
568564
{"control.max.time": {"$_internalExprLte": ISODate("2020-01-01T01:00:00Z")}},
569-
{"_id": {"$lte": ObjectId("5e0be100ffffffffffffffff")}},
570565
]
571566
}
572567
});
@@ -576,7 +571,6 @@ assert.sameMembers(buckets.getIndexes(), extraBucketIndexes.concat([
576571
$and: [
577572
{"control.max.time": {"$_internalExprGte": ISODate("2020-01-01T00:00:00Z")}},
578573
{"control.min.time": {"$_internalExprGte": ISODate("2019-12-31T23:00:00Z")}},
579-
{"_id": {"$gte": ObjectId("5e0bd2f00000000000000000")}},
580574
]
581575
}
582576
});
@@ -588,8 +582,6 @@ assert.sameMembers(buckets.getIndexes(), extraBucketIndexes.concat([
588582
{"control.min.time": {"$_internalExprGte": ISODate("2019-12-31T23:00:00Z")}},
589583
{"control.max.time": {"$_internalExprGte": ISODate("2020-01-01T00:00:00Z")}},
590584
{"control.max.time": {"$_internalExprLte": ISODate("2020-01-01T01:00:00Z")}},
591-
{"_id": {"$lte": ObjectId("5e0be100ffffffffffffffff")}},
592-
{"_id": {"$gte": ObjectId("5e0bd2f00000000000000000")}},
593585
]
594586
}
595587
});

0 commit comments

Comments
 (0)