Skip to content

Commit 41a745d

Browse files
shenyunlongshenyunlong
authored
[Test] add hbase rowkey TTL test cases for seconary partition scenario (#175)
1 parent fa52199 commit 41a745d

File tree

3 files changed

+201
-25
lines changed

3 files changed

+201
-25
lines changed

‎src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -216,8 +216,8 @@ public static void testMultiCFGetImpl(Map.Entry<String, List<String>> entry) thr
216216
Result r = hTable.get(get);
217217
Assert.assertEquals(tableNames.size() * columns.length, r.raw().length);
218218
int cur = 0;
219-
for (int i = 0; i < columns.length; i++) {
220-
for (String tableName : tableNames) {
219+
for (String tableName : tableNames) {
220+
for (int i = 0; i < columns.length; i++) {
221221
AssertKeyValue(key, columns[i], lastTs, latestValue, r.raw()[cur]);
222222
cur++;
223223
}
@@ -246,10 +246,11 @@ public static void testMultiCFGetImpl(Map.Entry<String, List<String>> entry) thr
246246
Result r = hTable.get(get);
247247
Assert.assertEquals(tableNames.size() * columns.length * ts.length, r.raw().length);
248248
int cur = 0;
249-
for (int i = 0; i < columns.length; i++) {
250-
for (String tableName : tableNames) {
251-
for (int k = ts.length-1; k >= 0; k--) {
252-
AssertKeyValue(key, columns[i], ts[k], values[k], r.raw()[cur]);
249+
for (String tableName : tableNames) {
250+
String family = getColumnFamilyName(tableName);
251+
for (int i = 0; i < columns.length; i++) {
252+
for (int k = ts.length - 1; k >= 0; k--) {
253+
AssertKeyValue(key, family, columns[i], ts[k], values[k], r.raw()[cur]);
253254
cur++;
254255
}
255256
}
@@ -264,9 +265,10 @@ public static void testMultiCFGetImpl(Map.Entry<String, List<String>> entry) thr
264265
Result r = hTable.get(get);
265266
Assert.assertEquals(tableNames.size() * columns.length, r.raw().length);
266267
int cur = 0;
267-
for (int i = 0; i < columns.length; i++) {
268-
for (String tableName : tableNames) {
269-
AssertKeyValue(key, columns[i], ts[1], values[1], r.raw()[cur]);
268+
for (String tableName : tableNames) {
269+
String family = getColumnFamilyName(tableName);
270+
for (int i = 0; i < columns.length; i++) {
271+
AssertKeyValue(key, family, columns[i], ts[1], values[1], r.raw()[cur]);
270272
cur++;
271273
}
272274
}
@@ -282,9 +284,10 @@ public static void testMultiCFGetImpl(Map.Entry<String, List<String>> entry) thr
282284
Result r = hTable.get(get);
283285
Assert.assertEquals(tableNames.size() * columns.length, r.raw().length);
284286
int cur = 0;
285-
for (int i = 0; i < columns.length; i++) {
286-
for (String tableName : tableNames) {
287-
AssertKeyValue(key, columns[i], ts[0], values[0], r.raw()[cur]);
287+
for (String tableName : tableNames) {
288+
String family = getColumnFamilyName(tableName);
289+
for (int i = 0; i < columns.length; i++) {
290+
AssertKeyValue(key, family, columns[i], ts[0], values[0], r.raw()[cur]);
288291
cur++;
289292
}
290293
}

‎src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java

Lines changed: 176 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public void prepareCase() throws Exception {
4646
}
4747

4848
public static void testTTLImpl(List<String> tableNames) throws Exception {
49-
closeTTLExecute();
49+
disableTTL();
5050
// 0. prepare data
5151
String keys[] = {"putKey1", "putKey2", "putKey3"};
5252
String endKey = "putKey4";
@@ -94,7 +94,8 @@ public static void testTTLImpl(List<String> tableNames) throws Exception {
9494
}
9595

9696
// 4. open ttl knob to delete expired hbase data
97-
openTTLExecute();
97+
enableTTL();
98+
triggerTTL();
9899

99100
// 5. check util expired hbase data is deleted by ttl tasks
100101
checkUtilTimeout(()-> {
@@ -110,11 +111,11 @@ public static void testTTLImpl(List<String> tableNames) throws Exception {
110111
}
111112

112113
// 6. close ttl knob
113-
closeTTLExecute();
114+
disableTTL();
114115
}
115116

116117
public static void testMultiCFTTLImpl(Map<String, List<String>> group2tableNames) throws Exception {
117-
closeTTLExecute();
118+
disableTTL();
118119
List<String> allTableNames = group2tableNames.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
119120
// 0. prepare data
120121
String keys[] = {"putKey1", "putKey2", "putKey3"};
@@ -127,7 +128,6 @@ public static void testMultiCFTTLImpl(Map<String, List<String>> group2tableNames
127128
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName);
128129
hTable.init();
129130
List<String> tableNames = entry.getValue();
130-
131131
for (String tableName : tableNames) {
132132
String family = getColumnFamilyName(tableName);
133133
for (String key : keys) {
@@ -168,7 +168,8 @@ public static void testMultiCFTTLImpl(Map<String, List<String>> group2tableNames
168168
}
169169

170170
// 4. open ttl knob to delete expired hbase data
171-
openTTLExecute();
171+
enableTTL();
172+
triggerTTL();
172173

173174
// 5. check util expired hbase data is deleted by ttl tasks
174175
checkUtilTimeout(()-> {
@@ -184,7 +185,159 @@ public static void testMultiCFTTLImpl(Map<String, List<String>> group2tableNames
184185
}
185186

186187
// 6. close ttl knob
187-
closeTTLExecute();
188+
disableTTL();
189+
}
190+
191+
void testRowkeyTTL(List<String> tableNames, Boolean useScan, Boolean isReversed) throws Exception {
192+
disableTTL();
193+
// 0. prepare data
194+
String keys[] = {"putKey1", "putKey2", "putKey3"};
195+
String endKey = "putKey4";
196+
String columns[] = {"putColumn1", "putColumn2"};
197+
String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"};
198+
for (String tableName : tableNames) {
199+
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName));
200+
hTable.init();
201+
String family = getColumnFamilyName(tableName);
202+
for (String key : keys) {
203+
for (String column : columns) {
204+
for (int i = 0; i < values.length; i++) {
205+
Put put = new Put(toBytes(key));
206+
put.add(family.getBytes(), column.getBytes(), values[i].getBytes());
207+
hTable.put(put);
208+
}
209+
}
210+
}
211+
}
212+
213+
// 1. sleep util data expired
214+
sleep(12000);
215+
216+
// 2. enable kv ttl
217+
enableTTL();
218+
219+
// 3. SQL can scan expired but not delete yet hbase data
220+
for (String tableName : tableNames) {
221+
Assert.assertEquals(keys.length * columns.length * values.length,
222+
getSQLTableRowCnt(tableName));
223+
}
224+
225+
// 4. use Hbase scan/get expired data to trigger ttl
226+
for (String tableName : tableNames) {
227+
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName));
228+
hTable.init();
229+
String family = getColumnFamilyName(tableName);
230+
if (useScan) {
231+
Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes());
232+
scan.addFamily(family.getBytes());
233+
scan.setReversed(isReversed);
234+
ResultScanner scanner = hTable.getScanner(scan);
235+
List<Cell> cells = getCellsFromScanner(scanner);
236+
assertEquals(0, cells.size());
237+
} else {
238+
for (String key : keys) {
239+
Get get = new Get(key.getBytes());
240+
get.addFamily(family.getBytes());
241+
Result result = hTable.get(get);
242+
assertEquals(0, result.rawCells().length);
243+
}
244+
}
245+
}
246+
247+
// 5. wait to disable
248+
checkUtilTimeout(()-> {
249+
try {
250+
Boolean passed = true;
251+
for (int i = 0; passed && i < tableNames.size(); i++) {
252+
if (getSQLTableRowCnt(tableNames.get(i)) > 0) {
253+
passed = false;
254+
}
255+
}
256+
return passed;
257+
} catch (Exception e) {
258+
throw new RuntimeException(e);
259+
}
260+
}, 50000, 3000);
261+
262+
// 6. disable ttl
263+
disableTTL();
264+
}
265+
266+
void testMultiCFRowkeyTTL(Map<String, List<String>> group2tableNames, Boolean useScan, Boolean isReversed) throws Exception {
267+
disableTTL();
268+
List<String> allTableNames = group2tableNames.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
269+
// 0. prepare data
270+
String keys[] = {"putKey1", "putKey2", "putKey3"};
271+
String endKey = "putKey4";
272+
String columns[] = {"putColumn1", "putColumn2"};
273+
String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"};
274+
for (Map.Entry<String, List<String>> entry : group2tableNames.entrySet()) {
275+
String groupName = getTableName(entry.getKey());
276+
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName);
277+
hTable.init();
278+
for (String tableName : entry.getValue()) {
279+
String family = getColumnFamilyName(tableName);
280+
for (String key : keys) {
281+
for (String column : columns) {
282+
for (int i = 0; i < values.length; i++) {
283+
Put put = new Put(toBytes(key));
284+
put.add(family.getBytes(), column.getBytes(), values[i].getBytes());
285+
hTable.put(put);
286+
}
287+
}
288+
}
289+
}
290+
}
291+
292+
// 1. sleep util data expired
293+
sleep(12000);
294+
295+
// 2. enable kv ttl
296+
enableTTL();
297+
298+
// 3. SQL can scan expired but not delete yet hbase data
299+
for (String tableName : allTableNames) {
300+
Assert.assertEquals(keys.length * columns.length * values.length,
301+
getSQLTableRowCnt(tableName));
302+
}
303+
304+
// 4. use Hbase scan expired data to trigger ttl
305+
for (Map.Entry<String, List<String>> entry : group2tableNames.entrySet()) {
306+
String groupName = getTableName(entry.getKey());
307+
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName);
308+
hTable.init();
309+
if (useScan) {
310+
Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes());
311+
scan.setReversed(isReversed);
312+
ResultScanner scanner = hTable.getScanner(scan);
313+
List<Cell> cells = getCellsFromScanner(scanner);
314+
assertEquals(0, cells.size());
315+
} else {
316+
for (String key : keys) {
317+
Get get = new Get(key.getBytes());
318+
Result result = hTable.get(get);
319+
assertEquals(0, result.rawCells().length);
320+
}
321+
}
322+
}
323+
324+
// 5. wait to disable
325+
checkUtilTimeout(()-> {
326+
try {
327+
Boolean passed = true;
328+
for (int i = 0; passed && i < allTableNames.size(); i++) {
329+
if (getSQLTableRowCnt(allTableNames.get(i)) > 0) {
330+
passed = false;
331+
}
332+
}
333+
return passed;
334+
} catch (Exception e) {
335+
throw new RuntimeException(e);
336+
}
337+
}, 70000, 5000);
338+
339+
// 6. disable ttl
340+
disableTTL();
188341
}
189342

190343

@@ -197,4 +350,20 @@ public void testTTL() throws Throwable {
197350
public void testMultiCFTTL() throws Throwable {
198351
testMultiCFTTLImpl(group2tableNames);
199352
}
353+
354+
@Test
355+
public void testRowkeyTTL() throws Exception {
356+
testRowkeyTTL(tableNames, true, false);
357+
testRowkeyTTL(tableNames, false, false);
358+
// TODO: open the test after reverse scan is ok
359+
// testRowkeyTTL(tableNames, true);
360+
}
361+
362+
@Test
363+
public void testMultiCFRowkeyTTL() throws Exception {
364+
testMultiCFRowkeyTTL(group2tableNames, true, false);
365+
testMultiCFRowkeyTTL(group2tableNames, false, false);
366+
// TODO: open the test after reverse scan is ok
367+
// testMultiCFRowkeyTTL(group2tableNames, true, true);
368+
}
200369
}

‎src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -273,15 +273,19 @@ public static int getRunningNormalTTLTaskCnt() throws Exception {
273273
return rowCnt;
274274
}
275275

276-
public static void openTTLExecute() throws Exception {
276+
public static void enableTTL() throws Exception {
277277
Connection conn = ObHTableTestUtil.getConnection();
278-
String stmt1 = "ALTER SYSTEM set enable_kv_ttl = true;";
279-
String stmt2 = "ALTER SYSTEM trigger TTL;";
280-
conn.createStatement().execute(stmt1);
281-
conn.createStatement().execute(stmt2);
278+
String stmt = "ALTER SYSTEM set enable_kv_ttl = true;";
279+
conn.createStatement().execute(stmt);
280+
}
281+
282+
public static void triggerTTL() throws Exception {
283+
Connection conn = ObHTableTestUtil.getConnection();
284+
String stmt = "ALTER SYSTEM trigger TTL;";
285+
conn.createStatement().execute(stmt);
282286
}
283287

284-
public static void closeTTLExecute() throws Exception {
288+
public static void disableTTL() throws Exception {
285289
Connection conn = ObHTableTestUtil.getConnection();
286290
String stmt = "ALTER SYSTEM set enable_kv_ttl = false;";
287291
conn.createStatement().execute(stmt);

0 commit comments

Comments
 (0)