Skip to content

Commit c7ab71c

Browse files
WeiXinChanWeiXinChan
authored
Merge pull request #174 from stuBirdFly/secondary_part
add cell ttl test
2 parents 0153939 + b401a50 commit c7ab71c

16 files changed

+1267
-498
lines changed

‎src/main/java/com/alipay/oceanbase/hbase/OHTable.java

Lines changed: 57 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ public class OHTable implements HTableInterface {
139139
/**
140140
* the buffer of put request
141141
*/
142-
private final ArrayList<Put> writeBuffer = new ArrayList<>();
142+
private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
143143
/**
144144
* when the put request reach the write buffer size the do put will
145145
* flush commits automatically
@@ -1563,6 +1563,7 @@ public void flushCommits() throws IOException {
15631563
} catch (Exception e) {
15641564
logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush,
15651565
writeBuffer.size(), e);
1566+
throw new IOException("put table " + tableNameString + " error.", e);
15661567
} finally {
15671568
// mutate list so that it is empty for complete success, or contains
15681569
// only failed records results are returned in the same order as the
@@ -2023,69 +2024,97 @@ public static ObTableBatchOperation buildObTableBatchOperation(List<Mutation> ro
20232024
batch.setSamePropertiesNames(true);
20242025
return batch;
20252026
}
2027+
20262028
private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv,
20272029
ObTableOperationType operationType,
20282030
boolean isTableGroup, Long TTL) {
20292031
KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType());
2030-
com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, operationType, isTableGroup, TTL);
2032+
com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, operationType,
2033+
isTableGroup, TTL);
20312034
ObNewRange range = new ObNewRange();
20322035
ObTableQuery tableQuery = new ObTableQuery();
2033-
tableQuery.setObKVParams(buildOBKVParams((Scan)null));
2036+
tableQuery.setObKVParams(buildOBKVParams((Scan) null));
20342037
ObHTableFilter filter = null;
20352038
switch (kvType) {
20362039
case Delete:
20372040
if (kv.getTimestamp() == Long.MAX_VALUE) {
2038-
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin()));
2039-
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax()));
2041+
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),
2042+
ObObj.getMin()));
2043+
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(),
2044+
ObObj.getMax()));
20402045
filter = buildObHTableFilter(null, null, 1, kv.getQualifier());
2041-
} else {
2042-
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),ObObj.getMin()));
2043-
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax()));
2044-
filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), 1, kv.getQualifier());
2046+
} else {
2047+
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),
2048+
ObObj.getMin()));
2049+
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(),
2050+
ObObj.getMax()));
2051+
filter = buildObHTableFilter(null,
2052+
new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), 1,
2053+
kv.getQualifier());
20452054
}
20462055
break;
20472056
case DeleteColumn:
20482057
if (kv.getTimestamp() == Long.MAX_VALUE) {
2049-
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin()));
2050-
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax()));
2058+
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),
2059+
ObObj.getMin()));
2060+
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(),
2061+
ObObj.getMax()));
20512062
filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, kv.getQualifier());
20522063
} else {
2053-
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin()));
2054-
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax()));
2055-
filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier());
2064+
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),
2065+
ObObj.getMin()));
2066+
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(),
2067+
ObObj.getMax()));
2068+
filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1),
2069+
Integer.MAX_VALUE, kv.getQualifier());
20562070
}
20572071
break;
20582072
case DeleteFamily:
20592073
if (kv.getTimestamp() == Long.MAX_VALUE) {
2060-
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin()));
2061-
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax()));
2074+
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),
2075+
ObObj.getMin()));
2076+
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(),
2077+
ObObj.getMax()));
20622078
if (!isTableGroup) {
20632079
filter = buildObHTableFilter(null, null, Integer.MAX_VALUE);
20642080
} else {
2065-
filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, kv.getQualifier());
2081+
filter = buildObHTableFilter(null, null, Integer.MAX_VALUE,
2082+
kv.getQualifier());
20662083
}
20672084
} else {
2068-
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin()));
2069-
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax()));
2085+
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),
2086+
ObObj.getMin()));
2087+
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(),
2088+
ObObj.getMax()));
20702089
if (!isTableGroup) {
2071-
filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE);
2090+
filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1),
2091+
Integer.MAX_VALUE);
20722092
} else {
2073-
filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier());
2093+
filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1),
2094+
Integer.MAX_VALUE, kv.getQualifier());
20742095
}
20752096
}
20762097
break;
20772098
case DeleteFamilyVersion:
20782099
if (kv.getTimestamp() == Long.MAX_VALUE) {
2079-
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin()));
2080-
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax()));
2100+
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),
2101+
ObObj.getMin()));
2102+
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(),
2103+
ObObj.getMax()));
20812104
filter = buildObHTableFilter(null, null, Integer.MAX_VALUE);
20822105
} else {
2083-
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin()));
2084-
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax()));
2106+
range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),
2107+
ObObj.getMin()));
2108+
range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(),
2109+
ObObj.getMax()));
20852110
if (!isTableGroup) {
2086-
filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), Integer.MAX_VALUE);
2111+
filter = buildObHTableFilter(null,
2112+
new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1),
2113+
Integer.MAX_VALUE);
20872114
} else {
2088-
filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier());
2115+
filter = buildObHTableFilter(null,
2116+
new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1),
2117+
Integer.MAX_VALUE, kv.getQualifier());
20892118
}
20902119
}
20912120
break;
@@ -2096,8 +2125,7 @@ private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv,
20962125
tableQuery.addKeyRange(range);
20972126
return new QueryAndMutate(tableQuery, tableMutation);
20982127
}
2099-
2100-
2128+
21012129
private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv,
21022130
ObTableOperationType operationType,
21032131
boolean isTableGroup, Long TTL) {

‎src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java

Lines changed: 59 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,7 @@
2323
import com.alipay.oceanbase.hbase.util.TableTemplateManager;
2424
import org.apache.hadoop.hbase.client.*;
2525
import org.apache.hadoop.hbase.util.Bytes;
26-
import org.junit.Before;
27-
import org.junit.BeforeClass;
28-
import org.junit.AfterClass;
29-
import org.junit.Test;
26+
import org.junit.*;
3027

3128
import java.io.IOException;
3229
import java.util.*;
@@ -38,12 +35,12 @@
3835
import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*;
3936
import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH;
4037
import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES;
41-
import static org.junit.Assert.assertEquals;
42-
import static org.junit.Assert.assertTrue;
38+
import static org.junit.Assert.*;
4339

4440
public class OHTableSecondaryPartAppendTest {
4541
private static List<String> tableNames = new LinkedList<String>();
46-
private static Map<String, List<String>> group2tableNames = null;
42+
private static List<String> series_tables = new LinkedList<String>();
43+
private static Map<String, List<String>> group2tableNames = new LinkedHashMap<String, List<String>>();
4744

4845
@BeforeClass
4946
public static void before() throws Exception {
@@ -56,7 +53,7 @@ public static void before() throws Exception {
5653
@AfterClass
5754
public static void finish() throws Exception {
5855
closeDistributedExecute();
59-
// dropTables(tableNames, group2tableNames);
56+
dropTables(tableNames, group2tableNames);
6057
}
6158

6259
@Before
@@ -152,6 +149,7 @@ private static void testAppendBorder(String tableName) throws Exception {
152149
a.add(FAMILY, QUALIFIERS[2], randomBytes);
153150
try {
154151
hTable.append(a);
152+
fail("unexpect error, too long data should fail");
155153
} catch (IOException e) {
156154
assertTrue(e.getCause().getMessage().contains("Data too long for column 'V'"));
157155
}
@@ -199,6 +197,47 @@ private static void testAppendCon(String tableName) throws Exception {
199197
hTable.close();
200198
}
201199

200+
private static void testAppendMultiCF(Map.Entry<String, List<String>> entry) throws Exception {
201+
String groupName = getTableName(entry.getKey());
202+
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName);
203+
hTable.init();
204+
List<String> tableNames = entry.getValue();
205+
String column = "appColumn";
206+
byte[] ROW = "appendKey".getBytes();
207+
byte[] v = "a".getBytes();
208+
Append append = new Append(ROW);
209+
for (String tableName : tableNames) {
210+
byte[] FAMILY = getColumnFamilyName(tableName).getBytes();
211+
append.add(FAMILY, column.getBytes(), v);
212+
}
213+
try {
214+
hTable.append(append);
215+
fail("unexpect error, append should not support multi cf");
216+
} catch (Exception e) {
217+
assertTrue(e.getMessage().contains("multi family is not supported"));
218+
}
219+
hTable.close();
220+
}
221+
222+
private static void testAppendSeires(String tableName) throws Exception {
223+
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName));
224+
hTable.init();
225+
byte[] FAMILY = getColumnFamilyName(tableName).getBytes();
226+
String column = "appColumn";
227+
byte[] ROW = "appendKey".getBytes();
228+
byte[] v = "a".getBytes();
229+
Append append = new Append(ROW);
230+
append.add(FAMILY, column.getBytes(), v);
231+
try {
232+
hTable.append(append);
233+
fail("unexpect error, append should not support series table");
234+
} catch (Exception e) {
235+
assertTrue(e.getCause().getMessage()
236+
.contains("query and mutate with hbase series type not supported"));
237+
}
238+
hTable.close();
239+
}
240+
202241
@Test
203242
public void testAppend() throws Throwable {
204243
FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppend);
@@ -213,4 +252,16 @@ public void testBorderAppend() throws Throwable {
213252
public void testAppendConcurrency() throws Throwable {
214253
FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendCon);
215254
}
255+
256+
@Test
257+
public void testAppendMultiCF() throws Throwable {
258+
FOR_EACH(group2tableNames, OHTableSecondaryPartAppendTest::testAppendMultiCF);
259+
}
260+
261+
@Test
262+
public void testAppendSeires() throws Throwable {
263+
createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, group2tableNames, true);
264+
FOR_EACH(series_tables, OHTableSecondaryPartAppendTest::testAppendSeires);
265+
dropTables(series_tables, group2tableNames);
266+
}
216267
}

‎src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,14 @@
3535
import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH;
3636
import static org.apache.hadoop.hbase.util.Bytes.toBytes;
3737

38-
3938
public class OHTableSecondaryPartBatchGetTest {
4039
private static List<String> tableNames = new LinkedList<String>();
41-
private static Map<String, List<String>> group2tableNames = new LinkedHashMap<>();
42-
40+
private static Map<String, List<String>> group2tableNames = new LinkedHashMap<String, List<String>>();
4341

4442
@BeforeClass
4543
public static void before() throws Exception {
4644
openDistributedExecute();
47-
for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) {
45+
for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_AND_SERIES_TABLES) {
4846
createTables(type, tableNames, group2tableNames, true);
4947
}
5048
}
@@ -60,7 +58,6 @@ public void prepareCase() throws Exception {
6058
truncateTables(tableNames, group2tableNames);
6159
}
6260

63-
6461
public static void testBatchGetImpl(String tableName) throws Exception {
6562
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName));
6663
hTable.init();
@@ -98,7 +95,7 @@ public static void testBatchGetImpl(String tableName) throws Exception {
9895
}
9996
}
10097
}
101-
98+
10299
@Test
103100
public void testBatchGet() throws Throwable {
104101
FOR_EACH(tableNames, OHTableSecondaryPartBatchGetTest::testBatchGetImpl);

‎src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
* #L%
1616
*/
1717

18-
1918
package com.alipay.oceanbase.hbase.secondary;
2019

2120
import com.alipay.oceanbase.hbase.OHTableClient;
@@ -39,9 +38,8 @@
3938
import static org.apache.hadoop.hbase.util.Bytes.toBytes;
4039

4140
public class OHTableSecondaryPartBatchTest {
42-
private static List<String> tableNames = new LinkedList<String>();
43-
private static Map<String, List<String>> group2tableNames = new LinkedHashMap<>();
44-
41+
private static List<String> tableNames = new LinkedList<String>();
42+
private static Map<String, List<String>> group2tableNames = new LinkedHashMap<String, List<String>>();
4543

4644
@BeforeClass
4745
public static void before() throws Exception {
@@ -431,8 +429,8 @@ public static void testBatchGetImpl(String tableName) throws Exception {
431429
hTable.close();
432430
}
433431

434-
435-
public static void testMultiCFBatchGetImpl(Map.Entry<String, List<String>> entry) throws Exception {
432+
public static void testMultiCFBatchGetImpl(Map.Entry<String, List<String>> entry)
433+
throws Exception {
436434
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey()));
437435
hTable.init();
438436
// prepare data
@@ -461,5 +459,4 @@ public void testMultiCFGet() throws Throwable {
461459
FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchGetImpl);
462460
truncateTables(ObHTableTestUtil.getConnection(), group2tableNames);
463461
}
464-
465462
}

0 commit comments

Comments
 (0)