Skip to content

Commit 1f119a7

Browse files
LuciferYangyhuang-db
LuciferYang
authored and
yhuang-db
committed
[SPARK-52386][CORE][TESTS] Refactor the HistoryServerSuite to support regenerating the expectation.json files using SPARK_GENERATE_GOLDEN_FILES=1
### What changes were proposed in this pull request? This por refactors the `HistoryServerSuite` to support regenerating the `expectation.json` files using `SPARK_GENERATE_GOLDEN_FILES=1`, and refreshes the `expectation.json` files with the new approach: ``` SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "core/testOnly org.apache.spark.deploy.history.RocksDBBackendHistoryServerSuite" ``` Meanwhile, this or also cleans up some methods which is no longer needed. ### Why are the changes needed? According to the documentation, the `HistoryServerSuite.main` method was previously used to regenerate golden files. However, there are currently 3 issues: 1. The `HistoryServerSuite.main` method generates the `expectation.json` file in the `target/scala-2.13/test-classes/HistoryServerExpectations` directory, rather than directly refreshing the content in the `core/src/test/resources/HistoryServerExpectations` directory. 2. The previous `expectation.json` files was likely based on data that had undergone re-formatting, rather than directly storing the file generated by `HistoryServerSuite.main` in the `HistoryServerExpectations` directory. 3. The `HistoryServerSuite` generates golden files in a different way compared to other tests, and it does not support the `SPARK_GENERATE_GOLDEN_FILES` parameter, which is somewhat inconvenient. Therefore, this pr enables support for the `SPARK_GENERATE_GOLDEN_FILES`. When `SPARK_GENERATE_GOLDEN_FILES=1`, it directly refreshes the content in the `core/src/test/resources/HistoryServerExpectations` directory. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? - Pass GitHub Actions ### Was this patch authored or co-authored using generative AI tooling? No Closes apache#51072 from LuciferYang/refactor-HistoryServerSuite. Lead-authored-by: yangjie01 <yangjie01@baidu.com> Co-authored-by: YangJie <yangjie01@baidu.com> Signed-off-by: yangjie01 <yangjie01@baidu.com>
1 parent 69f279c commit 1f119a7

File tree

47 files changed

+2379
-2640
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+2379
-2640
lines changed

‎core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json

Lines changed: 5 additions & 285 deletions
Large diffs are not rendered by default.

‎core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
"sparkUser" : "lijunqing",
1010
"completed" : true,
1111
"appSparkVersion" : "3.3.0-SNAPSHOT",
12-
"startTimeEpoch" : 1642039450519,
1312
"endTimeEpoch" : 1642039536564,
14-
"lastUpdatedEpoch" : 0
13+
"lastUpdatedEpoch" : 0,
14+
"startTimeEpoch" : 1642039450519
1515
} ]
1616
}, {
1717
"id" : "application_1628109047826_1317105",
@@ -24,9 +24,9 @@
2424
"sparkUser" : "john",
2525
"completed" : true,
2626
"appSparkVersion" : "3.1.1.119",
27-
"startTimeEpoch" : 1628637895333,
2827
"endTimeEpoch" : 1628638170208,
29-
"lastUpdatedEpoch" : 0
28+
"lastUpdatedEpoch" : 0,
29+
"startTimeEpoch" : 1628637895333
3030
} ]
3131
}, {
3232
"id" : "app-20200706201101-0003",
@@ -40,8 +40,8 @@
4040
"completed" : true,
4141
"appSparkVersion" : "3.1.0-SNAPSHOT",
4242
"endTimeEpoch" : 1594091824231,
43-
"startTimeEpoch" : 1594091460235,
44-
"lastUpdatedEpoch" : 0
43+
"lastUpdatedEpoch" : 0,
44+
"startTimeEpoch" : 1594091460235
4545
} ]
4646
}, {
4747
"id" : "application_1578436911597_0052",
@@ -55,8 +55,8 @@
5555
"completed" : true,
5656
"appSparkVersion" : "3.0.0-SNAPSHOT",
5757
"endTimeEpoch" : 1578764802615,
58-
"startTimeEpoch" : 1578764662851,
59-
"lastUpdatedEpoch" : 0
58+
"lastUpdatedEpoch" : 0,
59+
"startTimeEpoch" : 1578764662851
6060
} ]
6161
}, {
6262
"id" : "application_1555004656427_0144",
@@ -69,9 +69,9 @@
6969
"sparkUser" : "tgraves",
7070
"completed" : true,
7171
"appSparkVersion" : "3.0.0-SNAPSHOT",
72-
"startTimeEpoch" : 1562101337180,
72+
"endTimeEpoch" : 1562101355974,
7373
"lastUpdatedEpoch" : 0,
74-
"endTimeEpoch" : 1562101355974
74+
"startTimeEpoch" : 1562101337180
7575
} ]
7676
}, {
7777
"id" : "application_1553914137147_0018",
@@ -84,9 +84,9 @@
8484
"sparkUser" : "systest",
8585
"completed" : true,
8686
"appSparkVersion" : "3.0.0-SNAPSHOT",
87-
"startTimeEpoch" : 1554755984286,
8887
"endTimeEpoch" : 1554756046454,
89-
"lastUpdatedEpoch" : 0
88+
"lastUpdatedEpoch" : 0,
89+
"startTimeEpoch" : 1554755984286
9090
} ]
9191
}, {
9292
"id" : "application_1516285256255_0012",
@@ -99,9 +99,9 @@
9999
"sparkUser" : "attilapiros",
100100
"completed" : true,
101101
"appSparkVersion" : "2.3.0-SNAPSHOT",
102+
"endTimeEpoch" : 1516300707938,
102103
"lastUpdatedEpoch" : 0,
103-
"startTimeEpoch" : 1516300235119,
104-
"endTimeEpoch" : 1516300707938
104+
"startTimeEpoch" : 1516300235119
105105
} ]
106106
}, {
107107
"id" : "app-20180109111548-0000",
@@ -114,9 +114,9 @@
114114
"sparkUser" : "attilapiros",
115115
"completed" : true,
116116
"appSparkVersion" : "2.3.0-SNAPSHOT",
117+
"endTimeEpoch" : 1515493477606,
117118
"lastUpdatedEpoch" : 0,
118-
"startTimeEpoch" : 1515492942372,
119-
"endTimeEpoch" : 1515493477606
119+
"startTimeEpoch" : 1515492942372
120120
} ]
121121
}, {
122122
"id" : "app-20161116163331-0000",
@@ -129,9 +129,9 @@
129129
"sparkUser" : "jose",
130130
"completed" : true,
131131
"appSparkVersion" : "2.1.0-SNAPSHOT",
132+
"endTimeEpoch" : 1479335620587,
132133
"lastUpdatedEpoch" : 0,
133-
"startTimeEpoch" : 1479335609916,
134-
"endTimeEpoch" : 1479335620587
134+
"startTimeEpoch" : 1479335609916
135135
} ]
136136
}, {
137137
"id" : "app-20161115172038-0000",
@@ -144,9 +144,9 @@
144144
"sparkUser" : "jose",
145145
"completed" : true,
146146
"appSparkVersion" : "2.1.0-SNAPSHOT",
147+
"endTimeEpoch" : 1479252138874,
147148
"lastUpdatedEpoch" : 0,
148-
"startTimeEpoch" : 1479252037079,
149-
"endTimeEpoch" : 1479252138874
149+
"startTimeEpoch" : 1479252037079
150150
} ]
151151
}, {
152152
"id" : "local-1430917381534",
@@ -159,9 +159,9 @@
159159
"sparkUser" : "irashid",
160160
"completed" : true,
161161
"appSparkVersion" : "1.4.0-SNAPSHOT",
162+
"endTimeEpoch" : 1430917391398,
162163
"lastUpdatedEpoch" : 0,
163-
"startTimeEpoch" : 1430917380893,
164-
"endTimeEpoch" : 1430917391398
164+
"startTimeEpoch" : 1430917380893
165165
} ]
166166
}, {
167167
"id" : "local-1430917381535",
@@ -175,9 +175,9 @@
175175
"sparkUser" : "irashid",
176176
"completed" : true,
177177
"appSparkVersion" : "1.4.0-SNAPSHOT",
178+
"endTimeEpoch" : 1430917380950,
178179
"lastUpdatedEpoch" : 0,
179-
"startTimeEpoch" : 1430917380893,
180-
"endTimeEpoch" : 1430917380950
180+
"startTimeEpoch" : 1430917380893
181181
}, {
182182
"attemptId" : "1",
183183
"startTime" : "2015-05-06T13:03:00.880GMT",
@@ -187,9 +187,9 @@
187187
"sparkUser" : "irashid",
188188
"completed" : true,
189189
"appSparkVersion" : "1.4.0-SNAPSHOT",
190+
"endTimeEpoch" : 1430917380890,
190191
"lastUpdatedEpoch" : 0,
191-
"startTimeEpoch" : 1430917380880,
192-
"endTimeEpoch" : 1430917380890
192+
"startTimeEpoch" : 1430917380880
193193
} ]
194194
}, {
195195
"id" : "local-1426533911241",
@@ -203,9 +203,9 @@
203203
"sparkUser" : "irashid",
204204
"completed" : true,
205205
"appSparkVersion" : "",
206+
"endTimeEpoch" : 1426633945177,
206207
"lastUpdatedEpoch" : 0,
207-
"startTimeEpoch" : 1426633910242,
208-
"endTimeEpoch" : 1426633945177
208+
"startTimeEpoch" : 1426633910242
209209
}, {
210210
"attemptId" : "1",
211211
"startTime" : "2015-03-16T19:25:10.242GMT",
@@ -215,9 +215,9 @@
215215
"sparkUser" : "irashid",
216216
"completed" : true,
217217
"appSparkVersion" : "",
218+
"endTimeEpoch" : 1426533945177,
218219
"lastUpdatedEpoch" : 0,
219-
"startTimeEpoch" : 1426533910242,
220-
"endTimeEpoch" : 1426533945177
220+
"startTimeEpoch" : 1426533910242
221221
} ]
222222
}, {
223223
"id" : "local-1425081759269",
@@ -230,9 +230,9 @@
230230
"sparkUser" : "irashid",
231231
"completed" : true,
232232
"appSparkVersion" : "",
233+
"endTimeEpoch" : 1425081766912,
233234
"lastUpdatedEpoch" : 0,
234-
"startTimeEpoch" : 1425081758277,
235-
"endTimeEpoch" : 1425081766912
235+
"startTimeEpoch" : 1425081758277
236236
} ]
237237
}, {
238238
"id" : "local-1422981780767",
@@ -245,9 +245,9 @@
245245
"sparkUser" : "irashid",
246246
"completed" : true,
247247
"appSparkVersion" : "",
248+
"endTimeEpoch" : 1422981788731,
248249
"lastUpdatedEpoch" : 0,
249-
"startTimeEpoch" : 1422981779720,
250-
"endTimeEpoch" : 1422981788731
250+
"startTimeEpoch" : 1422981779720
251251
} ]
252252
}, {
253253
"id" : "local-1422981759269",
@@ -260,8 +260,8 @@
260260
"sparkUser" : "irashid",
261261
"completed" : true,
262262
"appSparkVersion" : "",
263+
"endTimeEpoch" : 1422981766912,
263264
"lastUpdatedEpoch" : 0,
264-
"startTimeEpoch" : 1422981758277,
265-
"endTimeEpoch" : 1422981766912
265+
"startTimeEpoch" : 1422981758277
266266
} ]
267267
} ]

‎core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,12 @@
3535
"shuffleReadRecords" : 0,
3636
"shuffleCorruptMergedBlockChunks" : 0,
3737
"shuffleMergedFetchFallbackCount" : 0,
38-
"shuffleMergedRemoteBlocksFetched": 0,
39-
"shuffleMergedLocalBlocksFetched": 0,
40-
"shuffleMergedRemoteChunksFetched": 0,
41-
"shuffleMergedLocalChunksFetched": 0,
42-
"shuffleMergedRemoteBytesRead": 0,
43-
"shuffleMergedLocalBytesRead": 0,
38+
"shuffleMergedRemoteBlocksFetched" : 0,
39+
"shuffleMergedLocalBlocksFetched" : 0,
40+
"shuffleMergedRemoteChunksFetched" : 0,
41+
"shuffleMergedLocalChunksFetched" : 0,
42+
"shuffleMergedRemoteBytesRead" : 0,
43+
"shuffleMergedLocalBytesRead" : 0,
4444
"shuffleRemoteReqsDuration" : 0,
4545
"shuffleMergedRemoteReqsDuration" : 0,
4646
"shuffleWriteBytes" : 0,
@@ -53,8 +53,6 @@
5353
"accumulatorUpdates" : [ ],
5454
"killedTasksSummary" : { },
5555
"resourceProfileId" : 0,
56-
"isShufflePushEnabled" : false,
57-
"shuffleMergersCount" : 0,
5856
"peakExecutorMetrics" : {
5957
"JVMHeapMemory" : 0,
6058
"JVMOffHeapMemory" : 0,
@@ -79,7 +77,9 @@
7977
"TotalGCTime" : 0,
8078
"ConcurrentGCCount" : 0,
8179
"ConcurrentGCTime" : 0
82-
}
80+
},
81+
"isShufflePushEnabled" : false,
82+
"shuffleMergersCount" : 0
8383
}, {
8484
"status" : "COMPLETE",
8585
"stageId" : 1,
@@ -117,12 +117,12 @@
117117
"shuffleReadRecords" : 0,
118118
"shuffleCorruptMergedBlockChunks" : 0,
119119
"shuffleMergedFetchFallbackCount" : 0,
120-
"shuffleMergedRemoteBlocksFetched": 0,
121-
"shuffleMergedLocalBlocksFetched": 0,
122-
"shuffleMergedRemoteChunksFetched": 0,
123-
"shuffleMergedLocalChunksFetched": 0,
124-
"shuffleMergedRemoteBytesRead": 0,
125-
"shuffleMergedLocalBytesRead": 0,
120+
"shuffleMergedRemoteBlocksFetched" : 0,
121+
"shuffleMergedLocalBlocksFetched" : 0,
122+
"shuffleMergedRemoteChunksFetched" : 0,
123+
"shuffleMergedLocalChunksFetched" : 0,
124+
"shuffleMergedRemoteBytesRead" : 0,
125+
"shuffleMergedLocalBytesRead" : 0,
126126
"shuffleRemoteReqsDuration" : 0,
127127
"shuffleMergedRemoteReqsDuration" : 0,
128128
"shuffleWriteBytes" : 13180,
@@ -135,8 +135,6 @@
135135
"accumulatorUpdates" : [ ],
136136
"killedTasksSummary" : { },
137137
"resourceProfileId" : 0,
138-
"isShufflePushEnabled" : false,
139-
"shuffleMergersCount" : 0,
140138
"peakExecutorMetrics" : {
141139
"JVMHeapMemory" : 0,
142140
"JVMOffHeapMemory" : 0,
@@ -161,7 +159,9 @@
161159
"TotalGCTime" : 0,
162160
"ConcurrentGCCount" : 0,
163161
"ConcurrentGCTime" : 0
164-
}
162+
},
163+
"isShufflePushEnabled" : false,
164+
"shuffleMergersCount" : 0
165165
}, {
166166
"status" : "COMPLETE",
167167
"stageId" : 0,
@@ -199,12 +199,12 @@
199199
"shuffleReadRecords" : 0,
200200
"shuffleCorruptMergedBlockChunks" : 0,
201201
"shuffleMergedFetchFallbackCount" : 0,
202-
"shuffleMergedRemoteBlocksFetched": 0,
203-
"shuffleMergedLocalBlocksFetched": 0,
204-
"shuffleMergedRemoteChunksFetched": 0,
205-
"shuffleMergedLocalChunksFetched": 0,
206-
"shuffleMergedRemoteBytesRead": 0,
207-
"shuffleMergedLocalBytesRead": 0,
202+
"shuffleMergedRemoteBlocksFetched" : 0,
203+
"shuffleMergedLocalBlocksFetched" : 0,
204+
"shuffleMergedRemoteChunksFetched" : 0,
205+
"shuffleMergedLocalChunksFetched" : 0,
206+
"shuffleMergedRemoteBytesRead" : 0,
207+
"shuffleMergedLocalBytesRead" : 0,
208208
"shuffleRemoteReqsDuration" : 0,
209209
"shuffleMergedRemoteReqsDuration" : 0,
210210
"shuffleWriteBytes" : 0,
@@ -217,8 +217,6 @@
217217
"accumulatorUpdates" : [ ],
218218
"killedTasksSummary" : { },
219219
"resourceProfileId" : 0,
220-
"isShufflePushEnabled" : false,
221-
"shuffleMergersCount" : 0,
222220
"peakExecutorMetrics" : {
223221
"JVMHeapMemory" : 0,
224222
"JVMOffHeapMemory" : 0,
@@ -243,5 +241,7 @@
243241
"TotalGCTime" : 0,
244242
"ConcurrentGCCount" : 0,
245243
"ConcurrentGCTime" : 0
246-
}
244+
},
245+
"isShufflePushEnabled" : false,
246+
"shuffleMergersCount" : 0
247247
} ]

0 commit comments

Comments
 (0)