From 46e71b782457fc2aa739fabba691fc791dcc7d84 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 9 Jun 2025 17:32:30 +0800 Subject: [PATCH 01/15] hbase client 2.0.0-alpha4 --- pom.xml | 4 +- .../com/alipay/oceanbase/hbase/OHTable.java | 137 ++---- .../alipay/oceanbase/hbase/OHTableClient.java | 65 ++- .../alipay/oceanbase/hbase/OHTablePool.java | 171 ++++++- .../hbase/filter/HBaseFilterUtils.java | 18 - .../hbase/result/ClientStreamScanner.java | 1 - .../alipay/oceanbase/hbase/util/OHAdmin.java | 157 ++++--- .../hbase/util/OHBufferedMutatorImpl.java | 68 +-- .../hbase/util/OHConnectionConfiguration.java | 19 - .../oceanbase/hbase/util/OHRegionLoad.java | 63 +++ .../hbase/util/OHRegionLoadExecutor.java | 71 +++ .../oceanbase/hbase/util/OHRegionMetrics.java | 121 ----- .../hbase/util/OHRegionMetricsExecutor.java | 71 --- .../hbase/HTableMultiCFTestBase.java | 49 +- .../oceanbase/hbase/HTableTestBase.java | 443 +++--------------- .../oceanbase/hbase/OHConnectionTest.java | 2 +- .../hbase/OHTableAdminInterfaceTest.java | 48 +- .../alipay/oceanbase/hbase/OHTableTest.java | 1 - .../hbase/filter/HBaseFilterUtilsTest.java | 9 - 19 files changed, 606 insertions(+), 912 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoad.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoadExecutor.java delete mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java delete mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java diff --git a/pom.xml b/pom.xml index 3b8fbf94..7797f2cf 100644 --- a/pom.xml +++ b/pom.xml @@ -45,8 +45,8 @@ - 2.7.7 - 2.0.6 + 3.1.0 + 2.0.0-alpha4 1.8 1.8 4.13.1 diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 106058eb..7253b23c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -620,8 +620,8 @@ private BatchOperation compatOldServerDel(final List actions, fin } else if (delete.getFamilyCellMap().size() > 1) { boolean has_delete_family = delete.getFamilyCellMap().entrySet().stream() .flatMap(entry -> entry.getValue().stream()).anyMatch( - kv -> kv.getType().getCode() == KeyValue.Type.DeleteFamily.getCode() || - kv.getType().getCode() == KeyValue.Type.DeleteFamilyVersion.getCode()); + kv -> kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode() || + kv.getTypeByte() == KeyValue.Type.DeleteFamilyVersion.getCode()); if (!has_delete_family) { return buildBatchOperation(tableNameString, Collections.singletonList(delete), true, @@ -1402,8 +1402,8 @@ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, Compa } @Override - public CheckAndMutateBuilder checkAndMutate(byte[] row, byte[] family) { - return new ObCheckAndMutateBuilderImpl(row, family); + public long getRpcTimeout(TimeUnit unit) { + return getRpcTimeout(); } private boolean checkAndMutation(byte[] row, byte[] family, byte[] qualifier, @@ -1723,6 +1723,11 @@ public int getOperationTimeout() { return operationTimeout; } + @Override + public long getOperationTimeout(TimeUnit unit) { + return getOperationTimeout(); + } + //todo @Override public void setRpcTimeout(int rpcTimeout) { @@ -1741,11 +1746,31 @@ public int getReadRpcTimeout() { return this.readRpcTimeout; } + @Override + public void setReadRpcTimeout(int readRpcTimeout) { + this.readRpcTimeout = readRpcTimeout; + } + @Override + public long getReadRpcTimeout(TimeUnit unit) { + return getReadRpcTimeout(); + } + + @Override + public long getWriteRpcTimeout(TimeUnit unit) { + return this.readRpcTimeout; + } + @Override public int getWriteRpcTimeout() { return this.writeRpcTimeout; } + @Override + public void setWriteRpcTimeout(int writeRpcTimeout) { + this.writeRpcTimeout = writeRpcTimeout; + } + + public void setRuntimeBatchExecutor(ExecutorService runtimeBatchExecutor) { this.obTableClient.setRuntimeBatchExecutor(runtimeBatchExecutor); } @@ -2016,7 +2041,7 @@ public static ObTableBatchOperation buildObTableBatchOperation(List ro private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv, ObTableOperationType operationType, boolean isTableGroup, byte[] family, Long TTL) { - KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType().getCode()); + KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getTypeByte()); com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, operationType, isTableGroup, family, TTL); if(isTableGroup) { @@ -2136,7 +2161,7 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, System.arraycopy(oldQualifier, 0, newQualifier, family.length + 1, oldQualifier.length); newCell = modifyQualifier(kv, newQualifier); } - Cell.Type kvType = kv.getType(); + KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getTypeByte()); switch (kvType) { case Put: String[] propertyColumns = V_COLUMNS; @@ -2182,7 +2207,7 @@ private KeyValue modifyQualifier(Cell original, byte[] newQualifier) { byte[] family = CellUtil.cloneFamily(original); byte[] value = CellUtil.cloneValue(original); long timestamp = original.getTimestamp(); - KeyValue.Type type = KeyValue.Type.codeToType(original.getType().getCode()); + KeyValue.Type type = KeyValue.Type.codeToType(original.getTypeByte()); // Create a new KeyValue with the modified qualifier return new KeyValue(row, family, newQualifier, timestamp, type, value); } @@ -2312,7 +2337,7 @@ private BatchOperation buildBatchOperation(String tableName, List public static ObTableOperation buildObTableOperation(Cell kv, ObTableOperationType operationType, Long TTL) { - Cell.Type kvType = kv.getType(); + KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getTypeByte()); String[] propertyColumns = V_COLUMNS; Object[] property = new Object[] { CellUtil.cloneValue(kv) }; if (TTL != Long.MAX_VALUE) { @@ -2449,7 +2474,7 @@ public Pair getStartEndKeys() throws IOException { return new Pair<>(getStartKeys(), getEndKeys()); } - private CompareFilter.CompareOp getCompareOp(CompareOperator cmpOp) { + public static CompareFilter.CompareOp getCompareOp(CompareOperator cmpOp) { switch (cmpOp) { case LESS: return CompareFilter.CompareOp.LESS; @@ -2467,96 +2492,4 @@ private CompareFilter.CompareOp getCompareOp(CompareOperator cmpOp) { return CompareFilter.CompareOp.NO_OP; } } - - private class ObCheckAndMutateBuilderImpl implements CheckAndMutateBuilder { - private final byte[] row; - private final byte[] family; - private byte[] qualifier; - private byte[] value; - private TimeRange timeRange; - private CompareOperator cmpOp; - - ObCheckAndMutateBuilderImpl(byte[] row, byte[] family) { - this.row = checkNotNull(row, "The provided row is null."); - this.family = checkNotNull(family, "The provided family is null."); - } - - @Override - public CheckAndMutateBuilder qualifier(byte[] qualifier) { - this.qualifier = checkNotNull( - qualifier, - "The provided qualifier is null. You could" - + " use an empty byte array, or do not call this method if you want a null qualifier."); - return this; - } - - @Override - public CheckAndMutateBuilder timeRange(TimeRange timeRange) { - this.timeRange = timeRange; - return this; - } - - @Override - public CheckAndMutateBuilder ifNotExists() { - this.cmpOp = CompareOperator.EQUAL; - this.value = null; - return this; - } - - @Override - public CheckAndMutateBuilder ifMatches(CompareOperator cmpOp, byte[] value) { - this.cmpOp = checkNotNull(cmpOp, "The provided cmpOp is null."); - this.value = checkNotNull(value, "The provided value is null."); - return this; - } - - @Override - public boolean thenPut(Put put) throws IOException { - checkCmpOp(); - RowMutations rowMutations = new RowMutations(row); - rowMutations.add(put); - try { - return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, - timeRange, rowMutations); - } catch (Exception e) { - logger.error(LCD.convert("01-00005"), rowMutations, tableNameString, e); - throw new IOException("checkAndMutate type table: " + tableNameString + " e.msg: " - + e.getMessage() + " error.", e); - } - } - - @Override - public boolean thenDelete(Delete delete) throws IOException { - checkCmpOp(); - RowMutations rowMutations = new RowMutations(row); - rowMutations.add(delete); - try { - return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, - timeRange, rowMutations); - } catch (Exception e) { - logger.error(LCD.convert("01-00005"), rowMutations, tableNameString, e); - throw new IOException("checkAndMutate type table: " + tableNameString + " e.msg: " - + e.getMessage() + " error.", e); - } - } - - @Override - public boolean thenMutate(RowMutations mutation) throws IOException { - checkCmpOp(); - try { - return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, - timeRange, mutation); - } catch (Exception e) { - logger.error(LCD.convert("01-00005"), mutation, tableNameString, e); - throw new IOException("checkAndMutate type table: " + tableNameString + " e.msg: " - + e.getMessage() + " error.", e); - } - } - - private void checkCmpOp() { - checkNotNull(this.cmpOp, - "The compare condition is null. Please use" - + " ifNotExists/ifEquals/ifMatches before executing the request"); - } - } -} \ No newline at end of file +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java index 8a2c2f89..89c7d35d 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java @@ -23,6 +23,7 @@ import com.google.protobuf.Service; import com.google.protobuf.ServiceException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompareOperator; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; @@ -35,8 +36,11 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import static com.alipay.oceanbase.hbase.OHTable.getCompareOp; + public class OHTableClient implements Table, Lifecycle { private byte[] tableName; private String tableNameString; @@ -174,9 +178,14 @@ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, } @Override - public CheckAndMutateBuilder checkAndMutate(byte[] row, byte[] family) { + public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, RowMutations mutation) throws IOException { + return checkAndMutate(row, family, qualifier, getCompareOp(op), value, mutation); + } + + @Override + public long getRpcTimeout(TimeUnit unit) { checkStatus(); - return ohTable.checkAndMutate(row, family); + return ohTable.getRpcTimeout(unit); } @Override @@ -197,6 +206,48 @@ public void setRpcTimeout(int i) { ohTable.setRpcTimeout(i); } + @Override + public long getReadRpcTimeout(TimeUnit unit) { + checkStatus(); + return ohTable.getReadRpcTimeout(unit); + } + + @Override + public int getReadRpcTimeout() { + checkStatus(); + return ohTable.getReadRpcTimeout(); + } + + @Override + public void setReadRpcTimeout(int readRpcTimeout) { + checkStatus(); + ohTable.setReadRpcTimeout(readRpcTimeout); + } + + @Override + public long getWriteRpcTimeout(TimeUnit unit) { + checkStatus(); + return ohTable.getWriteRpcTimeout(unit); + } + + @Override + public int getWriteRpcTimeout() { + checkStatus(); + return ohTable.getWriteRpcTimeout(); + } + + @Override + public void setWriteRpcTimeout(int writeRpcTimeout) { + checkStatus(); + ohTable.setWriteRpcTimeout(writeRpcTimeout); + } + + @Override + public long getOperationTimeout(TimeUnit unit) { + checkStatus(); + return ohTable.getOperationTimeout(unit); + } + @Override public int getRpcTimeout() { checkStatus(); @@ -323,6 +374,11 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, return ohTable.checkAndPut(row, family, qualifier, compareOp, value, put); } + @Override + public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, Put put) throws IOException { + return checkAndPut(row, family, qualifier, getCompareOp(op), value, put); + } + @Override public void delete(Delete delete) throws IOException { checkStatus(); @@ -350,6 +406,11 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, return ohTable.checkAndDelete(row, family, qualifier, compareOp, value, delete); } + @Override + public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, Delete delete) throws IOException { + return ohTable.checkAndDelete(row, family, qualifier, getCompareOp(op), value, delete); + } + // Not support. @Override public void mutateRow(RowMutations rm) throws IOException { diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java index 2214321e..b8e1c6b9 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java @@ -25,6 +25,7 @@ import com.google.protobuf.Service; import com.google.protobuf.ServiceException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompareOperator; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; @@ -42,7 +43,9 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import static com.alipay.oceanbase.hbase.OHTable.getCompareOp; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT; @@ -802,6 +805,30 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, return table.checkAndPut(row, family, qualifier, compareOp, value, put); } + /** + * Atomically checks if a row/family/qualifier value matches the expected + * value. If it does, it adds the put. If the passed value is null, the check + * is for the lack of column (ie: non-existence) + *

+ * The expected value argument of this call is on the left and the current + * value of the cell is on the right side of the comparison operator. + *

+ * Ie. eg. GREATER operator means expected value > existing <=> add the put. + * + * @param row to check + * @param family column family to check + * @param qualifier column qualifier to check + * @param op comparison operator to use + * @param value the expected value + * @param put data to put if check succeeds + * @return true if the new put was executed, false otherwise + * @throws IOException e + */ + @Override + public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, Put put) throws IOException { + return checkAndPut(row, family, qualifier, getCompareOp(op), value, put); + } + @Override public void delete(Delete delete) throws IOException { table.delete(delete); @@ -825,9 +852,28 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, return table.checkAndDelete(row, family, qualifier, compareOp, value, delete); } + /** + * Atomically checks if a row/family/qualifier value matches the expected + * value. If it does, it adds the delete. If the passed value is null, the + * check is for the lack of column (ie: non-existence) + *

+ * The expected value argument of this call is on the left and the current + * value of the cell is on the right side of the comparison operator. + *

+ * Ie. eg. GREATER operator means expected value > existing <=> add the delete. + * + * @param row to check + * @param family column family to check + * @param qualifier column qualifier to check + * @param op comparison operator to use + * @param value the expected value + * @param delete data to delete if check succeeds + * @return true if the new delete was executed, false otherwise + * @throws IOException e + */ @Override - public CheckAndMutateBuilder checkAndMutate(byte[] row, byte[] family) { - return table.checkAndMutate(row, family); + public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, Delete delete) throws IOException { + return checkAndDelete(row, family, qualifier, getCompareOp(op), value, delete); } @Override @@ -935,6 +981,44 @@ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, return table.checkAndMutate(row, family, qualifier, compareOp, value, mutations); } + /** + * Atomically checks if a row/family/qualifier value matches the expected value. + * If it does, it performs the row mutations. If the passed value is null, the check + * is for the lack of column (ie: non-existence) + *

+ * The expected value argument of this call is on the left and the current + * value of the cell is on the right side of the comparison operator. + *

+ * Ie. eg. GREATER operator means expected value > existing <=> perform row mutations. + * + * @param row to check + * @param family column family to check + * @param qualifier column qualifier to check + * @param op the comparison operator + * @param value the expected value + * @param mutation mutations to perform if check succeeds + * @return true if the new put was executed, false otherwise + * @throws IOException e + */ + @Override + public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, RowMutations mutation) throws IOException { + return checkAndMutate(row, family, qualifier, getCompareOp(op), value, mutation); + } + + /** + * Get timeout of each rpc request in this Table instance. It will be overridden by a more + * specific rpc timeout config such as readRpcTimeout or writeRpcTimeout. + * + * @param unit the unit of time the timeout to be represented in + * @return rpc timeout in the specified time unit + * @see #getReadRpcTimeout(TimeUnit) + * @see #getWriteRpcTimeout(TimeUnit) + */ + @Override + public long getRpcTimeout(TimeUnit unit) { + return getRpcTimeout(); + } + @Override public void setOperationTimeout(int i) { table.setOperationTimeout(i); @@ -950,6 +1034,89 @@ public void setRpcTimeout(int i) { table.setRpcTimeout(i); } + /** + * Get timeout of each rpc read request in this Table instance. + * + * @param unit the unit of time the timeout to be represented in + * @return read rpc timeout in the specified time unit + */ + @Override + public long getReadRpcTimeout(TimeUnit unit) { + return table.getReadRpcTimeout(unit); + } + + /** + * Get timeout (millisecond) of each rpc read request in this Table instance. + * + * @deprecated since 2.0 and will be removed in 3.0 version + * use {@link #getReadRpcTimeout(TimeUnit)} instead + */ + @Override + public int getReadRpcTimeout() { + return table.getReadRpcTimeout(); + } + + /** + * Set timeout (millisecond) of each rpc read request in operations of this Table instance, will + * override the value of hbase.rpc.read.timeout in configuration. + * If a rpc read request waiting too long, it will stop waiting and send a new request to retry + * until retries exhausted or operation timeout reached. + * + * @param readRpcTimeout the timeout for read rpc request in milliseconds + * @deprecated since 2.0.0, use {@link TableBuilder#setReadRpcTimeout} instead + */ + @Override + public void setReadRpcTimeout(int readRpcTimeout) { + table.setReadRpcTimeout(readRpcTimeout); + } + + /** + * Get timeout of each rpc write request in this Table instance. + * + * @param unit the unit of time the timeout to be represented in + * @return write rpc timeout in the specified time unit + */ + @Override + public long getWriteRpcTimeout(TimeUnit unit) { + return table.getWriteRpcTimeout(unit); + } + + /** + * Get timeout (millisecond) of each rpc write request in this Table instance. + * + * @deprecated since 2.0 and will be removed in 3.0 version + * use {@link #getWriteRpcTimeout(TimeUnit)} instead + */ + @Override + public int getWriteRpcTimeout() { + return table.getWriteRpcTimeout(); + } + + /** + * Set timeout (millisecond) of each rpc write request in operations of this Table instance, will + * override the value of hbase.rpc.write.timeout in configuration. + * If a rpc write request waiting too long, it will stop waiting and send a new request to retry + * until retries exhausted or operation timeout reached. + * + * @param writeRpcTimeout the timeout for write rpc request in milliseconds + * @deprecated since 2.0.0, use {@link TableBuilder#setWriteRpcTimeout} instead + */ + @Override + public void setWriteRpcTimeout(int writeRpcTimeout) { + table.setWriteRpcTimeout(writeRpcTimeout); + } + + /** + * Get timeout of each operation in Table instance. + * + * @param unit the unit of time the timeout to be represented in + * @return operation rpc timeout in the specified time unit + */ + @Override + public long getOperationTimeout(TimeUnit unit) { + return table.getOperationTimeout(unit); + } + @Override public int getRpcTimeout() { return table.getRpcTimeout(); diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index a791d7f7..6849249b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -76,8 +76,6 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (FuzzyRowFilter) filter); } else if (filter instanceof TimestampsFilter) { toParseableByteArray(byteStream, (TimestampsFilter) filter); - } else if (filter instanceof ColumnValueFilter) { - toParseableByteArray(byteStream, (ColumnValueFilter) filter); } else if (filter instanceof MultiRowRangeFilter) { toParseableByteArray(byteStream, (MultiRowRangeFilter) filter); } else if (filter instanceof InclusiveStopFilter) { @@ -354,22 +352,6 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Times byteStream.write(')'); } - // ColumnValueFilter('cf','q') - private static void toParseableByteArray(ByteArrayOutputStream byteStream, - ColumnValueFilter filter) throws IOException { - byteStream.write(filter.getClass().getSimpleName().getBytes()); - byteStream.write('('); - byteStream.write("'".getBytes()); - writeBytesWithEscape(byteStream, filter.getFamily()); - byteStream.write("','".getBytes()); - writeBytesWithEscape(byteStream, filter.getQualifier()); - byteStream.write("',".getBytes()); - byteStream.write(toParseableByteArray(filter.getCompareOperator())); - byteStream.write(','); - toParseableByteArray(byteStream, filter.getComparator()); - byteStream.write(')'); - } - // MultiRowRangeFilter('a',true,'b',false,'c',true,'d',false); private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultiRowRangeFilter filter) throws IOException { diff --git a/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java b/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java index dca76d20..be880fa0 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java +++ b/src/main/java/com/alipay/oceanbase/hbase/result/ClientStreamScanner.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; -import org.mortbay.util.SingletonList; import org.slf4j.Logger; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index add7434c..ad12c90e 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -5,6 +5,7 @@ import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.rpc.exception.ObTableTransportException; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.commons.cli.Option; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; @@ -16,7 +17,6 @@ import org.apache.hadoop.hbase.quotas.QuotaRetriever; import org.apache.hadoop.hbase.quotas.QuotaSettings; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; -import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; @@ -427,11 +427,6 @@ public void flushRegion(byte[] bytes) throws IOException { throw new FeatureNotSupportedException("does not support yet"); } - @Override - public void flushRegionServer(ServerName serverName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); - } - @Override public void compact(TableName tableName) throws IOException { throw new FeatureNotSupportedException("does not support yet"); @@ -482,23 +477,28 @@ public void majorCompactRegion(byte[] bytes, byte[] bytes1) throws IOException { throw new FeatureNotSupportedException("does not support yet"); } + /** + * Compact all regions on the region server. Asynchronous operation in that this method requests + * that a Compaction run and then it returns. It does not wait on the completion of Compaction + * (it can take a while). + * + * @param sn the region server name + * @param major if it's major compaction + * @throws IOException + * @throws InterruptedException + */ @Override - public void majorCompact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public void majorCompact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { + public void compactRegionServer(ServerName sn, boolean major) throws IOException, InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void compactRegionServer(ServerName serverName) throws IOException { + public void majorCompact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void majorCompactRegionServer(ServerName serverName) throws IOException { + public void majorCompact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } @@ -542,11 +542,6 @@ public boolean isBalancerEnabled() throws IOException { throw new FeatureNotSupportedException("does not support yet"); } - @Override - public CacheEvictionStats clearBlockCache(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); - } - @Override public boolean normalize() throws IOException { throw new FeatureNotSupportedException("does not support yet"); @@ -672,25 +667,64 @@ public void stopRegionServer(String s) throws IOException { throw new FeatureNotSupportedException("does not support yet"); } + /** + * Get whole cluster status, containing status about: + * 

+     * hbase version
+     * cluster id
+     * primary/backup master(s)
+     * master's coprocessors
+     * live/dead regionservers
+     * balancer
+     * regions in transition
+     *
+ * + * @return cluster status + * @throws IOException if a remote or network exception occurs + */ @Override - public ClusterMetrics getClusterMetrics(EnumSet enumSet) throws IOException { + public ClusterStatus getClusterStatus() throws IOException { throw new FeatureNotSupportedException("does not support yet"); } + /** + * Get cluster status with a set of {@link Option} to get desired status. + * + * @param options + * @return cluster status + * @throws IOException if a remote or network exception occurs + */ @Override - public List getRegionMetrics(ServerName serverName) throws IOException { + public ClusterStatus getClusterStatus(EnumSet options) throws IOException { throw new FeatureNotSupportedException("does not support yet"); } + /** + * Get {@link RegionLoad} of all regions hosted on a regionserver. + * + * @param serverName region server from which regionload is required. + * @return region load map of all regions hosted on a region server + * @throws IOException if a remote or network exception occurs + */ @Override - public List getRegionMetrics(ServerName serverName, TableName tableName) throws IOException { - if (tableName == null) { - throw new FeatureNotSupportedException("does not support tableName is null"); - } + public Map getRegionLoad(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + /** + * Get {@link RegionLoad} of all regions hosted on a regionserver for a table. + * + * @param serverName region server from which regionload is required. + * @param tableName get region load of regions belonging to the table + * @return region load map of all regions of a table hosted on a region server + * @throws IOException if a remote or network exception occurs + */ + @Override + public Map getRegionLoad(ServerName serverName, TableName tableName) throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHRegionMetricsExecutor executor = new OHRegionMetricsExecutor(tableClient); - return executor.getRegionMetrics(tableName.getNameAsString()); + OHRegionLoadExecutor executor = new OHRegionLoadExecutor(tableName.getNameAsString(), tableClient); + return executor.getRegionLoad(); } @Override @@ -807,6 +841,17 @@ public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCl throw new FeatureNotSupportedException("does not support yet"); } + /** + * Helper that delegates to getClusterStatus().getMasterCoprocessors(). + * + * @return an array of master coprocessors + * @see ClusterStatus#getMasterCoprocessors() + */ + @Override + public String[] getMasterCoprocessors() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + @Override public CompactionState getCompactionState(TableName tableName) throws IOException { throw new FeatureNotSupportedException("does not support yet"); @@ -992,11 +1037,6 @@ public QuotaRetriever getQuotaRetriever(QuotaFilter quotaFilter) throws IOExcept throw new FeatureNotSupportedException("does not support yet"); } - @Override - public List getQuota(QuotaFilter quotaFilter) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); - } - @Override public CoprocessorRpcChannel coprocessorService() { throw new FeatureNotSupportedException("does not support yet"); @@ -1022,28 +1062,27 @@ public List getSecurityCapabilities() throws IOException { throw new FeatureNotSupportedException("does not support yet"); } + /** + * Turn the Split or Merge switches on or off. + * + * @param enabled enabled or not + * @param synchronous If true, it waits until current split() call, if outstanding, to return. + * @param switchTypes switchType list {@link MasterSwitchType} + * @return Previous switch value array + */ @Override - public boolean splitSwitch(boolean b, boolean b1) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public boolean mergeSwitch(boolean b, boolean b1) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public boolean isSplitEnabled() throws IOException { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public boolean isMergeEnabled() throws IOException { + public boolean[] splitOrMergeEnabledSwitch(boolean enabled, boolean synchronous, MasterSwitchType... switchTypes) throws IOException { throw new FeatureNotSupportedException("does not support yet"); } + /** + * Query the current state of the switch. + * + * @param switchType + * @return true if the switch is enabled, false otherwise. + */ @Override - public void addReplicationPeer(String s, ReplicationPeerConfig replicationPeerConfig, boolean b) throws IOException { + public boolean splitOrMergeEnabledSwitch(MasterSwitchType switchType) throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -1072,16 +1111,6 @@ public void updateReplicationPeerConfig(String s, ReplicationPeerConfig replicat throw new FeatureNotSupportedException("does not support yet"); } - @Override - public void appendReplicationPeerTableCFs(String s, Map> map) throws ReplicationException, IOException { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public void removeReplicationPeerTableCFs(String s, Map> map) throws ReplicationException, IOException { - throw new FeatureNotSupportedException("does not support yet"); - } - @Override public List listReplicationPeers() throws IOException { throw new FeatureNotSupportedException("does not support yet"); @@ -1127,6 +1156,16 @@ public void clearCompactionQueues(ServerName serverName, Set set) throws throw new FeatureNotSupportedException("does not support yet"); } + /** + * List dead region servers. + * + * @return List of dead region servers. + */ + @Override + public List listDeadServers() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + @Override public List clearDeadServers(List list) throws IOException { throw new FeatureNotSupportedException("does not support yet"); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index d6e63d40..d4ca4aef 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -19,11 +19,11 @@ import com.alipay.oceanbase.hbase.OHTable; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import java.io.IOException; @@ -59,7 +59,7 @@ public class OHBufferedMutatorImpl implements BufferedMutator { private final AtomicLong writeBufferPeriodicFlushTimeoutMs = new AtomicLong( 0); private final AtomicLong writeBufferPeriodicFlushTimerTickMs = new AtomicLong( - MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS); + 0); private Timer writeBufferPeriodicFlushTimer = null; private final long writeBufferSize; @@ -96,14 +96,6 @@ public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParam params.getOperationTimeout() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getOperationTimeout() : connectionConfig.getOperationTimeout()); - long newPeriodicFlushTimeoutMs = params.getWriteBufferPeriodicFlushTimeoutMs() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params - .getWriteBufferPeriodicFlushTimeoutMs() : connectionConfig - .getWriteBufferPeriodicFlushTimeoutMs(); - long newPeriodicFlushTimeIntervalMs = params.getWriteBufferPeriodicFlushTimerTickMs() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params - .getWriteBufferPeriodicFlushTimerTickMs() : connectionConfig - .getWriteBufferPeriodicFlushTimerTickMs(); - this.setWriteBufferPeriodicFlush(newPeriodicFlushTimeoutMs, newPeriodicFlushTimeIntervalMs); - this.writeBufferSize = params.getWriteBufferSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getWriteBufferSize() : connectionConfig.getWriteBufferSize(); this.maxKeyValueSize = params.getMaxKeyValueSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params @@ -210,41 +202,6 @@ public void timeTriggerForWriteBufferPeriodicFlush() { } } - /** - * set time for periodic flush timer - * @param timeoutMs control when to flush from collecting first mutation - * @param timerTickMs control time interval to trigger the timer - * */ - @Override - public synchronized void setWriteBufferPeriodicFlush(long timeoutMs, long timerTickMs) { - long originalTimeoutMs = this.writeBufferPeriodicFlushTimeoutMs.get(); - long originalTimeTickMs = this.writeBufferPeriodicFlushTimerTickMs.get(); - - writeBufferPeriodicFlushTimeoutMs.set(Math.max(0, timeoutMs)); - writeBufferPeriodicFlushTimerTickMs.set(Math.max( - MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, timerTickMs)); - - // if time parameters are updated, stop the old timer - if (writeBufferPeriodicFlushTimeoutMs.get() != originalTimeoutMs - || writeBufferPeriodicFlushTimerTickMs.get() != originalTimeTickMs) { - if (writeBufferPeriodicFlushTimer != null) { - writeBufferPeriodicFlushTimer.cancel(); - writeBufferPeriodicFlushTimer = null; - } - } - - if (writeBufferPeriodicFlushTimer == null && writeBufferPeriodicFlushTimeoutMs.get() > 0) { - writeBufferPeriodicFlushTimer = new Timer(true); - writeBufferPeriodicFlushTimer.schedule(new TimerTask() { - @Override - public void run() { - OHBufferedMutatorImpl.this.timeTriggerForWriteBufferPeriodicFlush(); - } - }, this.writeBufferPeriodicFlushTimerTickMs.get(), - this.writeBufferPeriodicFlushTimerTickMs.get()); - } - } - /** * Send the operations in the buffer to the servers. Does not wait for the server's answer. If * there is an error, either throw the error, or use the listener to deal with the error. @@ -303,21 +260,14 @@ private void execute(boolean flushAll) throws IOException { } } - /** - * reset the time parameters and cancel the timer (if exists) - * */ - @Override - public void disableWriteBufferPeriodicFlush() { - setWriteBufferPeriodicFlush(0, MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS); - } - + @Override public void close() throws IOException { if (closed) { return; } // reset timeout, timeTick and Timer - disableWriteBufferPeriodicFlush(); +// disableWriteBufferPeriodicFlush(); try { execute(true); } finally { @@ -347,16 +297,6 @@ public void flush() throws IOException { execute(true); } - @Override - public long getWriteBufferPeriodicFlushTimeoutMs() { - return writeBufferPeriodicFlushTimeoutMs.get(); - } - - @Override - public long getWriteBufferPeriodicFlushTimerTickMs() { - return writeBufferPeriodicFlushTimerTickMs.get(); - } - @Override public long getWriteBufferSize() { return this.writeBufferSize; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index 3c966a18..d5d404a6 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -25,10 +25,6 @@ import java.util.Properties; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; -import static org.apache.hadoop.hbase.client.ConnectionConfiguration.WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS; -import static org.apache.hadoop.hbase.client.ConnectionConfiguration.WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS_DEFAULT; -import static org.apache.hadoop.hbase.client.ConnectionConfiguration.WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS; -import static org.apache.hadoop.hbase.client.ConnectionConfiguration.WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS_DEFAULT; import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.hadoop.hbase.ipc.RpcClient.DEFAULT_SOCKET_TIMEOUT_CONNECT; import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; @@ -55,8 +51,6 @@ public class OHConnectionConfiguration { private final int readRpcTimeout; private final int writeRpcTimeout; private final int rpcConnectTimeout; - private final long writeBufferPeriodicFlushTimeoutMs; - private final long writeBufferPeriodicFlushTimerTickMs; private final int numRetries; public OHConnectionConfiguration(Configuration conf) { @@ -84,11 +78,6 @@ public OHConnectionConfiguration(Configuration conf) { HConstants.DEFAULT_HBASE_RPC_TIMEOUT); this.writeRpcTimeout = conf.getInt(HConstants.HBASE_RPC_WRITE_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); - this.writeBufferPeriodicFlushTimeoutMs = conf.getLong( - WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS, WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS_DEFAULT); - this.writeBufferPeriodicFlushTimerTickMs = conf.getLong( - WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, - WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS_DEFAULT); int rpcConnectTimeout = -1; if (conf.get(SOCKET_TIMEOUT_CONNECT) != null) { rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT_CONNECT, DEFAULT_SOCKET_TIMEOUT_CONNECT); @@ -205,14 +194,6 @@ public String getDatabase() { return this.database; } - public long getWriteBufferPeriodicFlushTimeoutMs() { - return this.writeBufferPeriodicFlushTimeoutMs; - } - - public long getWriteBufferPeriodicFlushTimerTickMs() { - return this.writeBufferPeriodicFlushTimerTickMs; - } - public int getNumRetries() { return this.numRetries; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoad.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoad.java new file mode 100644 index 00000000..697d9646 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoad.java @@ -0,0 +1,63 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.hbase.RegionLoad; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos; + +public class OHRegionLoad extends RegionLoad { + private final byte[] name; // tablet_name, id in String + private int storeFileSize; // tablet storage used in ssTable + private int memStoreSize; // tablet storage used in memTable + + public OHRegionLoad(byte[] name, int storeFileSize, int memStoreSize) { + super(null); + this.name = name; + this.storeFileSize = storeFileSize; + this.memStoreSize = memStoreSize; + } + + @Override + public byte[] getName() { + return name; + } + + /** + * @return the number of stores + */ + @Override + public int getStores() { + return 1; + } + + /** + * @return the number of storefiles + */ + @Override + public int getStorefiles() { + return 1; + } + + /** + * @return the total size of the storefiles, in MB + */ + @Override + public int getStorefileSizeMB() { + return memStoreSize + storeFileSize / 1024 / 1024; + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoadExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoadExecutor.java new file mode 100644 index 00000000..a799e41d --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoadExecutor.java @@ -0,0 +1,71 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.RegionLoad; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class OHRegionLoadExecutor extends AbstractObTableMetaExecutor> { + private final String tableName; + private final ObTableClient client; + OHRegionLoadExecutor(String tableName, ObTableClient client) { + this.tableName = tableName; + this.client = client; + } + + /** + * 解析元数据响应, 用户需要重写 + * + * @param response 元数据响应 + * @return 解析后的元数据对象 + * @throws IOException 如果解析失败 + */ + @Override + public Map parse(ObTableMetaResponse response) throws IOException { + Map regionLoadMap = new LinkedHashMap<>(); + JSONObject metrcisJSONObject = JSON.parseObject(response.getData()); + String tableGroupName = metrcisJSONObject.getString("tableName"); + JSONObject regionList = metrcisJSONObject.getJSONObject("regionList"); + List regions = regionList.getJSONArray("regions").toJavaList(Integer.class); + List memTableSizeList = regionList.getJSONArray("memTableSize").toJavaList(Integer.class); + List ssTableSizeList = regionList.getJSONArray("ssTableSize").toJavaList(Integer.class); + if (regions.isEmpty() || regions.size() != memTableSizeList.size() || memTableSizeList.size() != ssTableSizeList.size()) { + throw new IOException("size length has to be the same"); + } + for (int i = 0; i < regions.size(); ++i) { + String name_str = Integer.toString(regions.get(i)); + byte[] name = name_str.getBytes(); + OHRegionLoad load = new OHRegionLoad(name, ssTableSizeList.get(i) / (1024 * 1024), memTableSizeList.get(i) / (1024 * 1024)); + regionLoadMap.put(name, load); + } + return regionLoadMap; + } + + /** + * 获取元信息类型, 用户需要重写 + * + * @return 元信息类型 + */ + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return ObTableRpcMetaType.HTABLE_REGION_METRICS; + } + + public Map getRegionLoad() throws IOException { + final ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + final Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + return execute(client, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java deleted file mode 100644 index 8f161575..00000000 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.alipay.oceanbase.hbase.util; - -import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; -import org.apache.hadoop.hbase.RegionMetrics; -import org.apache.hadoop.hbase.Size; - -import java.util.Collections; -import java.util.Map; - -public class OHRegionMetrics implements RegionMetrics { - private final String tablegroup; - private final byte[] name; // tablet_name, id in String - private final Size storeFileSize; // tablet storage used in ssTable - private final Size memStoreSize; // tablet storage used in memTable - - OHRegionMetrics(String tablegroup, byte[] name, Size storeFileSize, Size memStoreSize) { - this.tablegroup = tablegroup; - this.name = name; - this.storeFileSize = storeFileSize; - this.memStoreSize = memStoreSize; - } - - public String getTablegroup() { - return tablegroup; - } - - @Override - public byte[] getRegionName() { - return name; - } - - @Override - public int getStoreCount() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public int getStoreFileCount() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public Size getStoreFileSize() { - return storeFileSize; - } - - @Override - public Size getMemStoreSize() { - return memStoreSize; - } - - @Override - public long getReadRequestCount() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public long getWriteRequestCount() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public long getFilteredReadRequestCount() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public Size getStoreFileIndexSize() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public Size getStoreFileRootLevelIndexSize() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public Size getStoreFileUncompressedDataIndexSize() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public Size getBloomFilterSize() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public long getCompactingCellCount() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public long getCompactedCellCount() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public long getCompletedSequenceId() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public Map getStoreSequenceId() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public Size getUncompressedStoreFileSize() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public float getDataLocality() { - throw new FeatureNotSupportedException("does not support yet"); - } - - @Override - public long getLastMajorCompactionTimestamp() { - throw new FeatureNotSupportedException("does not support yet"); - } -} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java deleted file mode 100644 index 430f568f..00000000 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.alipay.oceanbase.hbase.util; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; -import com.alipay.oceanbase.rpc.ObTableClient; -import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; -import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; -import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; -import org.apache.hadoop.hbase.RegionMetrics; -import org.apache.hadoop.hbase.Size; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class OHRegionMetricsExecutor extends AbstractObTableMetaExecutor> { - private final ObTableClient tableClient; - OHRegionMetricsExecutor(ObTableClient tableClient) { - this.tableClient = tableClient; - } - @Override - public ObTableRpcMetaType getMetaType() throws IOException { - return ObTableRpcMetaType.HTABLE_REGION_METRICS; - } - - /* - * { - tableName: "tablegroup_name", - regionList:{ - "regions": [200051, 200052, 200053, 200191, 200192, 200193, ...], - "memTableSize":[123, 321, 321, 123, 321, 321, ...], - "ssTableSize":[5122, 4111, 5661, 5122, 4111, 5661, ...] - } - } - * */ - @Override - public List parse(ObTableMetaResponse response) throws IOException { - List metricsList = new ArrayList<>(); - JSONObject metrcisJSONObject = JSON.parseObject(response.getData()); - String tableGroupName = metrcisJSONObject.getString("tableName"); - JSONObject regionList = metrcisJSONObject.getJSONObject("regionList"); - List regions = regionList.getJSONArray("regions").toJavaList(Integer.class); - List memTableSizeList = regionList.getJSONArray("memTableSize").toJavaList(Integer.class); - List ssTableSizeList = regionList.getJSONArray("ssTableSize").toJavaList(Integer.class); - if (regions.isEmpty() || regions.size() != memTableSizeList.size() || memTableSizeList.size() != ssTableSizeList.size()) { - throw new IOException("size length has to be the same"); - } - for (int i = 0; i < regions.size(); ++i) { - String name_str = Integer.toString(regions.get(i)); - byte[] name = name_str.getBytes(); - Size storeFileSize = new Size(((double) ssTableSizeList.get(i)) / (1024 * 1024) , Size.Unit.MEGABYTE); // The unit in original HBase is MEGABYTE, for us it is BYTE - Size memStoreSize = new Size(((double) memTableSizeList.get(i)) / (1024 * 1024), Size.Unit.MEGABYTE); // The unit in original HBase is MEGABYTE, for us it is BYTE - OHRegionMetrics ohRegionMetrics = new OHRegionMetrics(tableGroupName, name, storeFileSize, memStoreSize); - metricsList.add(ohRegionMetrics); - } - return metricsList; - } - - public List getRegionMetrics(String tableName) throws IOException { - ObTableMetaRequest request = new ObTableMetaRequest(); - request.setMetaType(getMetaType()); - Map requestData = new HashMap<>(); - requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); - request.setData(jsonData); - return execute(tableClient, request); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index a22bc769..2bc7e482 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -22,11 +22,13 @@ import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; +import org.checkerframework.checker.units.qual.C; import org.junit.*; import org.junit.rules.ExpectedException; @@ -626,51 +628,6 @@ public void testMultiColumnFamilyBufferedMutator() throws Exception { } Assert.assertEquals(0, count); - // test periodic flush - params.setWriteBufferPeriodicFlushTimeoutMs(100); - mutator = connection.getBufferedMutator(params); - while (true) { - for (int i = 0; i < rows; ++i) { - mutations.clear(); - Put put = new Put(toBytes(keys.get(i))); - put.addColumn(family1, family1_column1, family1_value); - put.addColumn(family1, family1_column2, family1_value); - put.addColumn(family1, family1_column3, family1_value); - put.addColumn(family2, family2_column1, family2_value); - put.addColumn(family3, family3_column1, family2_value); - put.addColumn(family3, family3_column2, family3_value); - mutations.add(put); - if (i % 3 == 0) { // 0, 3, 6, 9 - Delete delete = new Delete(toBytes(keys.get(i))); - delete.addFamily(family1); - delete.addFamily(family2); - mutations.add(delete); - } - mutator.mutate(mutations); - } - - get = new Get(toBytes("Key0")); - result = multiCfHTable.get(get); - if (!result.isEmpty()) { - break; - } - } - get = new Get(toBytes("Key2")); - get.setMaxVersions(); - result = multiCfHTable.get(get); - count = result.rawCells().length; - Assert.assertTrue(count > 0); - // test timer periodic flush - int lastUndealtCount = ((OHBufferedMutatorImpl) mutator).size(); - Thread.sleep(1000); - int currentUndealtCount = ((OHBufferedMutatorImpl) mutator).size(); - Assert.assertNotEquals(lastUndealtCount, currentUndealtCount); - // after periodic flush, all mutations will be committed - Assert.assertEquals(0, currentUndealtCount); - result = multiCfHTable.get(get); - int newCount = result.rawCells().length; - Assert.assertNotEquals(count, newCount); - // clean data mutations.clear(); for (String key : keys) { @@ -898,7 +855,7 @@ public void update(byte[] region, byte[] row, Result result) { // put + delete + get Get get2 = new Get("Key2".getBytes()); - get1.setMaxVersions(10); + get2.setMaxVersions(10); get2.addColumn(family1, family1_column1); batchLsit.clear(); batchLsit.add(delete); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4dd3e0f7..734de8a2 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.*; @@ -2096,202 +2097,6 @@ public void testFilter2() throws Exception { scanner.close(); } - @Test - public void testColumnValueFilter() throws Exception { - String key1 = "abab"; - String key2 = "abcc"; - String column1 = "c1"; - String column2 = "c2"; - String column3 = "c3"; - String column4 = "c4"; - String column5 = "c5"; - String value1 = "value1"; - String value2 = "value2"; - String family = "family1"; - Delete deleteKey1Family = new Delete(toBytes(key1)); - deleteKey1Family.addFamily(toBytes(family)); - - Delete deleteKey2Family = new Delete(toBytes(key2)); - deleteKey2Family.addFamily(toBytes(family)); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - - try { - Put putKey1Column1Value1 = new Put(toBytes(key1)); - putKey1Column1Value1.addColumn(toBytes(family), toBytes(column1), toBytes(value1)); - - Put putKey1Column1Value2 = new Put(toBytes(key1)); - putKey1Column1Value2.addColumn(toBytes(family), toBytes(column1), toBytes(value2)); - - Put putKey1Column2Value2 = new Put(toBytes(key1)); - putKey1Column2Value2.addColumn(toBytes(family), toBytes(column2), toBytes(value2)); - - Put putKey1Column2Value1 = new Put(toBytes(key1)); - putKey1Column2Value1.addColumn(toBytes(family), toBytes(column2), toBytes(value1)); - - Put putKey1Column3Value1 = new Put(toBytes(key1)); - putKey1Column3Value1.addColumn(toBytes(family), toBytes(column3), toBytes(value1)); - - Put putKey1Column4Value1 = new Put(toBytes(key1)); - putKey1Column4Value1.addColumn(toBytes(family), toBytes(column4), toBytes(value1)); - - Put putKey1Column5Value1 = new Put(toBytes(key1)); - putKey1Column5Value1.addColumn(toBytes(family), toBytes(column5), toBytes(value1)); - - Put putKey2Column1Value1 = new Put(toBytes(key2)); - putKey2Column1Value1.addColumn(toBytes(family), toBytes(column1), toBytes(value1)); - - Put putKey2Column1Value2 = new Put(toBytes(key2)); - putKey2Column1Value2.addColumn(toBytes(family), toBytes(column1), toBytes(value2)); - - Put putKey2Column2Value2 = new Put(toBytes(key2)); - putKey2Column2Value2.addColumn(toBytes(family), toBytes(column2), toBytes(value2)); - - Put putKey2Column2Value1 = new Put(toBytes(key2)); - putKey2Column2Value1.addColumn(toBytes(family), toBytes(column2), toBytes(value1)); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - tryPut(hTable, putKey1Column1Value1); - tryPut(hTable, putKey1Column1Value2); - tryPut(hTable, putKey1Column1Value1); - tryPut(hTable, putKey1Column2Value1); - tryPut(hTable, putKey1Column2Value2); - tryPut(hTable, putKey1Column2Value1); - tryPut(hTable, putKey1Column2Value2); - tryPut(hTable, putKey1Column3Value1); - tryPut(hTable, putKey1Column4Value1); - tryPut(hTable, putKey1Column5Value1); - tryPut(hTable, putKey2Column2Value1); - tryPut(hTable, putKey2Column2Value2); - - Scan scan; - scan = new Scan(); - scan.addFamily(family.getBytes()); - scan.setMaxVersions(10); - ColumnValueFilter filter = new ColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareOperator.EQUAL, Bytes.toBytes(value1)); - scan.setFilter(filter); - ResultScanner scanner = hTable.getScanner(scan); - - int res_count = 0; - for (Result result : scanner) { - for (Cell keyValue : result.rawCells()) { - System.out - .printf( - "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); - res_count += 1; - } - } - Assert.assertEquals(res_count, 3); - scanner.close(); - - scan = new Scan(); - scan.addFamily(family.getBytes()); - scan.setMaxVersions(10); - filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), - CompareOperator.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("value"))); - scan.setFilter(filter); - scanner = hTable.getScanner(scan); - - res_count = 0; - for (Result result : scanner) { - for (Cell keyValue : result.rawCells()) { - System.out - .printf( - "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); - res_count += 1; - } - } - Assert.assertEquals(res_count, 6); - scanner.close(); - - scan = new Scan(); - scan.addFamily(family.getBytes()); - scan.setMaxVersions(10); - filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), - CompareOperator.LESS, Bytes.toBytes(value1)); - scan.setFilter(filter); - scanner = hTable.getScanner(scan); - - res_count = 0; - for (Result result : scanner) { - for (Cell keyValue : result.rawCells()) { - System.out - .printf( - "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); - res_count += 1; - } - } - Assert.assertEquals(res_count, 3); - scanner.close(); - - scan = new Scan(); - scan.addFamily(family.getBytes()); - scan.setMaxVersions(10); - filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), - CompareOperator.NOT_EQUAL, Bytes.toBytes(value1)); - scan.setFilter(filter); - scanner = hTable.getScanner(scan); - - res_count = 0; - for (Result result : scanner) { - for (Cell keyValue : result.rawCells()) { - System.out - .printf( - "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); - res_count += 1; - } - } - Assert.assertEquals(res_count, 1); - scanner.close(); - - scan = new Scan(); - scan.addFamily(family.getBytes()); - scan.setMaxVersions(10); - filter = new ColumnValueFilter(Bytes.toBytes("ff"), Bytes.toBytes(column1), - CompareOperator.NOT_EQUAL, Bytes.toBytes(value1)); - scan.setFilter(filter); - scanner = hTable.getScanner(scan); - - res_count = 0; - for (Result result : scanner) { - for (Cell keyValue : result.rawCells()) { - System.out - .printf( - "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); - res_count += 1; - } - } - Assert.assertEquals(res_count, 0); - scanner.close(); - } finally { - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - } - } - @Test public void testFuzzyRowFilter() throws Exception { String key1 = "abab"; @@ -2825,7 +2630,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( - Bytes.toBytes(family), Bytes.toBytes(column1)); + Bytes.toBytes(family), Bytes.toBytes(column1)); get.setFilter(dependentColumnFilter); r = hTable.get(get); Assert.assertEquals(3, r.rawCells().length); @@ -2852,7 +2657,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(value2))); + new BinaryComparator(toBytes(value2))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(0, r.rawCells().length); @@ -2861,7 +2666,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value1))); + toBytes(value1))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(2, r.rawCells().length); @@ -2873,7 +2678,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(2, r.rawCells().length); @@ -2882,7 +2687,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); @@ -2890,8 +2695,8 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - valueFilter = new ValueFilter(CompareFilter.CompareOp.LESS, new BinaryComparator( - toBytes(value1))); + valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( + toBytes(value1))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(2, r.rawCells().length); @@ -2900,7 +2705,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, - new BinaryComparator(toBytes(value1))); + new BinaryComparator(toBytes(value1))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); @@ -2908,8 +2713,8 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - valueFilter = new ValueFilter(CompareFilter.CompareOp.LESS, new BinaryComparator( - toBytes(value3))); + valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( + toBytes(value3))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(0, r.rawCells().length); @@ -2931,7 +2736,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(column1))); + new BinaryComparator(toBytes(column1))); get.setFilter(qualifierFilter); r = hTable.get(get); Assert.assertEquals(3, r.rawCells().length); @@ -2940,7 +2745,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(column2))); + toBytes(column2))); get.setFilter(qualifierFilter); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); @@ -2948,8 +2753,8 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.LESS, - new BinaryComparator(toBytes(column1))); + qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER, + new BinaryComparator(toBytes(column1))); get.setFilter(qualifierFilter); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); @@ -2958,7 +2763,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, - new BinaryComparator(toBytes(column1))); + new BinaryComparator(toBytes(column1))); get.setFilter(qualifierFilter); r = hTable.get(get); Assert.assertEquals(7, r.rawCells().length); @@ -3002,7 +2807,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3012,7 +2817,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3022,7 +2827,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), false)); + .toBytes(column1), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3032,7 +2837,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false)); + .toBytes(column2), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3042,7 +2847,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2))); + .toBytes(column2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3052,7 +2857,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), true)); + .toBytes(column2), true)); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3062,8 +2867,8 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2)))); + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2)))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3073,8 +2878,8 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); - filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.LESS, - new BinaryComparator(toBytes(column2)))); + filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, + new BinaryComparator(toBytes(column2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3085,7 +2890,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(MUST_PASS_ONE); filterList.addFilter(new ColumnCountGetFilter(2)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(column2)))); + new BinaryComparator(toBytes(column2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3096,7 +2901,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(2)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(column2)))); + new BinaryComparator(toBytes(column2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3117,7 +2922,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(column2)))); + new BinaryComparator(toBytes(column2)))); filterList.addFilter(new ColumnCountGetFilter(2)); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3129,7 +2934,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(2)); filterList.addFilter(new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2)))); + toBytes(value2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -3188,7 +2993,7 @@ public void testGetFilter() throws Exception { // 任何一个版本满足则返回本行 SingleColumnValueFilter singleColumnValueFilter; singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3199,7 +3004,7 @@ public void testGetFilter() throws Exception { SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3209,7 +3014,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(4, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3219,7 +3024,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(0, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3230,7 +3035,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value1))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3241,7 +3046,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(0, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3252,7 +3057,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3263,7 +3068,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3274,7 +3079,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3291,7 +3096,7 @@ public void testGetFilter() throws Exception { tryPut(hTable, putKey1Column2Value2); valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); SkipFilter skipFilter = new SkipFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3301,7 +3106,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(2, r.rawCells().length); valueFilter = new ValueFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); skipFilter = new SkipFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3314,7 +3119,7 @@ public void testGetFilter() throws Exception { tryPut(hTable, putKey1Column2Value1); valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); skipFilter = new SkipFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3324,7 +3129,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(0, r.rawCells().length); valueFilter = new ValueFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); skipFilter = new SkipFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3349,7 +3154,7 @@ public void testGetFilter() throws Exception { WhileMatchFilter whileMatchFilter; valueFilter = new ValueFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); whileMatchFilter = new WhileMatchFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -5032,71 +4837,6 @@ public void testCheckAndPut() throws IOException, InterruptedException { r = hTable.get(get); Assert.assertEquals(3, r.rawCells().length); Assert.assertEquals("value1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); - - // test CheckAndMutateBuilder - Delete delete = new Delete(key.getBytes()); - delete.addFamily(family.getBytes()); - hTable.delete(delete); - r = hTable.get(get); - Assert.assertEquals(0, r.rawCells().length); - - put = new Put(key.getBytes()); - put.addColumn(family.getBytes(), column.getBytes(), value.getBytes()); - hTable.put(put); - get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - get.addColumn(family.getBytes(), column.getBytes()); - r = hTable.get(get); - Assert.assertEquals(1, r.rawCells().length); - - Table.CheckAndMutateBuilder builder = hTable.checkAndMutate(toBytes(key), toBytes(family)); - put = new Put(key.getBytes()); - put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); - ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenPut(put); - Assert.assertTrue(ret); - Put difFamPut = new Put(key.getBytes()); - difFamPut.addColumn("family_group".getBytes(), column.getBytes(), "value1".getBytes()); - Assert.assertThrows(IOException.class, () -> { - builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenPut(difFamPut); - }); - - ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.LESS, toBytes("value1")).thenPut(put); - Assert.assertFalse(ret); - ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value1")).thenPut(put); - Assert.assertTrue(ret); - ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.GREATER, toBytes("")) - .thenPut(put); - Assert.assertFalse(ret); - ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("")).thenPut(put); - Assert.assertFalse(ret); - - get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - get.addColumn(family.getBytes(), column.getBytes()); - r = hTable.get(get); - Assert.assertEquals(3, r.rawCells().length); - Assert.assertEquals("value1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); - - // test TimeRange - long t = System.currentTimeMillis(); - put = new Put(key.getBytes()); - put.addColumn(family.getBytes(), column.getBytes(), t, value.getBytes()); - put.addColumn(family.getBytes(), column.getBytes(), t + 3, "value1".getBytes()); - put.addColumn(family.getBytes(), column.getBytes(), t + 5, "value2".getBytes()); - hTable.put(put); - put = new Put(toBytes(key)); - put.addColumn(toBytes(family), toBytes(column), toBytes(value)); - TimeRange timeRange = new TimeRange(t + 1, t + 3); - ret = builder.qualifier(toBytes(column)).timeRange(timeRange).ifEquals(toBytes("value1")) - .thenPut(put); - Assert.assertFalse(ret); - timeRange = new TimeRange(t, t + 2); - ret = builder.qualifier(toBytes(column)).timeRange(timeRange).ifEquals(toBytes(value)) - .thenPut(put); - Assert.assertTrue(ret); } @Test @@ -5202,29 +4942,31 @@ public void testCheckAndDelete() throws IOException { // check delete column delete = new Delete(key.getBytes()); delete.addColumn(family.getBytes(), column.getBytes()); - Table.CheckAndMutateBuilder builder = hTable.checkAndMutate(toBytes(key), toBytes(family)); - ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenDelete(delete); + RowMutations rowMutations = new RowMutations(key.getBytes()); + rowMutations.add(delete); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column.getBytes(), + CompareFilter.CompareOp.EQUAL, value.getBytes(), rowMutations); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value6".getBytes()); hTable.put(put); - ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.LESS, toBytes("value5")).thenDelete(delete); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column.getBytes(), + CompareFilter.CompareOp.LESS, "value5".getBytes(), rowMutations); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value5".getBytes()); hTable.put(put); - ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value5")).thenDelete(delete); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column.getBytes(), + CompareFilter.CompareOp.LESS_OR_EQUAL, "value5".getBytes(), rowMutations); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); hTable.put(put); - ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.GREATER, toBytes("value1")) - .thenDelete(delete); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column.getBytes(), + CompareFilter.CompareOp.GREATER, "value1".getBytes(), rowMutations); Assert.assertFalse(ret); - ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value1")).thenDelete(delete); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column.getBytes(), + CompareFilter.CompareOp.GREATER_OR_EQUAL, "value1".getBytes(), rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); @@ -5246,7 +4988,7 @@ public void testCheckAndDelete() throws IOException { Assert.assertEquals(2, r.rawCells().length); delete = new Delete(key.getBytes()); delete.addColumns(family.getBytes(), column.getBytes()); - ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenDelete(delete); + ret = hTable.checkAndDelete(key.getBytes(), family.getBytes(), column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), delete); Assert.assertTrue(ret); get = new Get(key.getBytes()); get.setMaxVersions(Integer.MAX_VALUE); @@ -5267,28 +5009,6 @@ public void testCheckAndDelete() throws IOException { get.addFamily(family.getBytes()); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); - - // test TimeRange - put = new Put(key.getBytes()); - put.addColumn(family.getBytes(), column.getBytes(), value.getBytes()); - TimeRange timeRange = new TimeRange(t + 2, t + 4); - ret = builder.qualifier(toBytes(column)).timeRange(timeRange).ifEquals(toBytes(value)) - .thenPut(put); - Assert.assertFalse(ret); - timeRange = new TimeRange(t, t + 2); - ret = builder.qualifier(toBytes(column)).timeRange(timeRange).ifEquals(toBytes(value)) - .thenPut(put); - Assert.assertTrue(ret); - - delete = new Delete(key.getBytes()); - delete.addFamily(family.getBytes()); - ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenDelete(delete); - Assert.assertTrue(ret); - get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - get.addFamily(family.getBytes()); - r = hTable.get(get); - Assert.assertEquals(0, r.rawCells().length); } @Test @@ -5459,9 +5179,8 @@ public void testCheckAndMutate() throws IOException { rowMutations.add(put3); //put data - Table.CheckAndMutateBuilder builder = hTable.checkAndMutate(toBytes(key), - family.getBytes(StandardCharsets.UTF_8)); - ret = builder.qualifier(toBytes(column1)).ifNotExists().thenMutate(rowMutations); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.NOT_EQUAL, null, rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); @@ -5488,8 +5207,8 @@ public void testCheckAndMutate() throws IOException { rowMutations.add(put2); rowMutations.add(put3); // test LESS op - ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value1)) - .thenMutate(rowMutations); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.LESS, value1.getBytes(), rowMutations); Assert.assertFalse(ret); get = new Get(key.getBytes()); get.addFamily(family.getBytes()); @@ -5498,8 +5217,8 @@ public void testCheckAndMutate() throws IOException { Assert.assertEquals(6, r.rawCells().length); // test less op - ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.GREATER, toBytes(value2)) - .thenMutate(rowMutations); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.GREATER, value2.getBytes(), rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); get.addFamily(family.getBytes()); @@ -5526,16 +5245,16 @@ public void testCheckAndMutate() throws IOException { rowMutations.add(put3); // test NO_OP try { - builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.NO_OP, toBytes(value1)) - .thenMutate(rowMutations); + hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.NO_OP, value1.getBytes(), rowMutations); fail(); } catch (Exception e) { Assert.assertTrue(e.getMessage().contains("checkAndMutate")); } // test equal op - ret = builder.qualifier(toBytes(column1)).ifEquals(toBytes(value1)) - .thenMutate(rowMutations); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.EQUAL, value1.getBytes(), rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); get.addFamily(family.getBytes()); @@ -5557,8 +5276,8 @@ public void testCheckAndMutate() throws IOException { rowMutations = new RowMutations(key.getBytes()); rowMutations.add(delete1); rowMutations.add(put1); - ret = builder.qualifier(toBytes(column1)).ifEquals(toBytes(value1)) - .thenMutate(rowMutations); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.EQUAL, value1.getBytes(), rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); get.addColumn(family.getBytes(), column1.getBytes()); @@ -5571,25 +5290,6 @@ public void testCheckAndMutate() throws IOException { get.setMaxVersions(Integer.MAX_VALUE); r = hTable.get(get); Assert.assertEquals(10, r.rawCells().length); - - // check TimeRange - put1 = new Put(toBytes(key)); - put1.addColumn(toBytes(family), toBytes(column1), toBytes(value1)); - rowMutations = new RowMutations(key.getBytes()); - rowMutations.add(put1); - rowMutations.add(delete1); - TimeRange timeRange = new TimeRange(t + 1, t + 3); - ret = builder.qualifier(toBytes(column1)).timeRange(timeRange).ifEquals(toBytes(value1)) - .thenMutate(rowMutations); - Assert.assertFalse(ret); - timeRange = new TimeRange(t, t + 2); - ret = builder.qualifier(toBytes(column1)).timeRange(timeRange).ifEquals(toBytes(value1)) - .thenMutate(rowMutations); - Assert.assertTrue(ret); - - delete = new Delete(key.getBytes()); - delete.addFamily(family.getBytes()); - hTable.delete(delete); } @Test @@ -6152,6 +5852,7 @@ public void testFamilyBlank() throws Exception { Assert.assertTrue(e.getMessage().contains("does not exist")); } Put put = new Put(key.getBytes()); + expectedException.expect(NullPointerException.class); put.addColumn(null, null, value.getBytes()); try { hTable.put(put); @@ -6239,6 +5940,7 @@ public void testScannerMultiVersion() throws Exception { @Test public void testPutColumnFamilyNull() throws Exception { Put put1 = new Put(("key_c_f").getBytes()); + expectedException.expect(NullPointerException.class); put1.addColumn(null, ("column1").getBytes(), "value1_family_null".getBytes()); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("family is empty"); @@ -6276,8 +5978,9 @@ public void testPutColumnFamilyNotExists() throws Exception { @Test public void testGetColumnFamilyNull() throws Exception { Get get = new Get(("key_c_f").getBytes()); + expectedException.expect(NullPointerException.class); get.addFamily(null); - expectedException.expect(IllegalArgumentException.class); + expectedException.expect(FeatureNotSupportedException.class); expectedException.expectMessage("family is empty"); hTable.get(get); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 3e4046ce..d18d85de 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -1078,7 +1078,7 @@ public void testBufferedMutatorPeriodicFlush() throws Exception { params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); // set periodic flush timeout to enable Timer - params.setWriteBufferPeriodicFlushTimeoutMs(100); +// params.setWriteBufferPeriodicFlushTimeoutMs(100); // set thread pool long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 748d21c8..9e1963cb 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -17,7 +17,6 @@ package com.alipay.oceanbase.hbase; -import com.alipay.oceanbase.hbase.util.OHRegionMetrics; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.hbase.util.ResultSetPrinter; @@ -40,6 +39,7 @@ import java.util.LinkedList; import java.util.List; import java.util.concurrent.*; +import java.util.Map; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; import static org.apache.hadoop.hbase.util.Bytes.toBytes; @@ -429,7 +429,7 @@ private void batchGet(int rows, String tablegroup) throws Exception { } @Test - public void testAdminGetRegionMetrics() throws Exception { + public void testRegionLoad() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + @@ -510,27 +510,27 @@ public void testAdminGetRegionMetrics() throws Exception { // test tablegroup not existed IOException thrown = assertThrows(IOException.class, () -> { - admin.getRegionMetrics(null, TableName.valueOf("tablegroup_not_exists")); + admin.getRegionLoad(ServerName.valueOf("localhost,1,1"), TableName.valueOf("tablegroup_not_exists")); }); Assert.assertTrue(thrown.getCause() instanceof ObTableException); Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); - // test use serverName without tableName to get region metrics + // test use serverName without tableName to get region load assertThrows(FeatureNotSupportedException.class, () -> { - admin.getRegionMetrics(ServerName.valueOf("localhost,1,1")); + admin.getRegionLoad(ServerName.valueOf("localhost,1,1")); }); - // test single-thread getRegionMetrics after writing + // test single-thread getRegionLoad after writing batchInsert(100000, tablegroup1); // test ServerName is any string long start = System.currentTimeMillis(); - List metrics = admin.getRegionMetrics(ServerName.valueOf("localhost,1,1"), TableName.valueOf(tablegroup1)); + Map regionLoadMap = admin.getRegionLoad(ServerName.valueOf("localhost,1,1"), TableName.valueOf(tablegroup1)); long cost = System.currentTimeMillis() - start; - System.out.println("get region metrics time usage: " + cost + "ms, tablegroup: " + tablegroup1); - assertEquals(30, metrics.size()); + System.out.println("get region load time usage: " + cost + "ms, tablegroup: " + tablegroup1); + assertEquals(30, regionLoadMap.size()); - // test getRegionMetrics concurrently reading while writing + // test getRegionLoad concurrently reading while writing ExecutorService executorService = Executors.newFixedThreadPool(10); CountDownLatch latch = new CountDownLatch(100); List exceptionCatcher = new ArrayList<>(); @@ -539,25 +539,25 @@ public void testAdminGetRegionMetrics() throws Exception { executorService.submit(() -> { try { if (taskId % 2 == 1) { - List regionMetrics = null; - // test get regionMetrics from different namespaces + Map regionLoad = null; + // test get regionLoad from different namespaces if (taskId % 3 != 0) { long thrStart = System.currentTimeMillis(); - regionMetrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup1)); + regionLoad = admin.getRegionLoad(ServerName.valueOf("localhost,1,1"), TableName.valueOf(tablegroup1)); long thrCost = System.currentTimeMillis() - thrStart; - System.out.println("task: " + taskId + ", get region metrics time usage: " + thrCost + "ms, tablegroup: " + tablegroup1); - if (regionMetrics.size() != 30) { + System.out.println("task: " + taskId + ", get region load time usage: " + thrCost + "ms, tablegroup: " + tablegroup1); + if (regionLoad.size() != 30) { throw new ObTableGetException( - "the number of region metrics does not match the number of tablets, the number of region metrics: " + regionMetrics.size()); + "the number of region load does not match the number of tablets, the number of region load: " + regionLoad.size()); } } else { long thrStart = System.currentTimeMillis(); - regionMetrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup2)); + regionLoad = admin.getRegionLoad(ServerName.valueOf("localhost,1,1"), TableName.valueOf(tablegroup2)); long thrCost = System.currentTimeMillis() - thrStart; - System.out.println("task: " + taskId + ", get region metrics time usage: " + thrCost + "ms, tablegroup: " + tablegroup1); - if (regionMetrics.size() != 9) { + System.out.println("task: " + taskId + ", get region load time usage: " + thrCost + "ms, tablegroup: " + tablegroup1); + if (regionLoad.size() != 9) { throw new ObTableGetException( - "the number of region metrics does not match the number of tablets, the number of region metrics: " + regionMetrics.size()); + "the number of region load does not match the number of tablets, the number of region load: " + regionLoad.size()); } } } else { @@ -598,14 +598,14 @@ public void testAdminGetRegionMetrics() throws Exception { executorService.shutdownNow(); Assert.assertTrue(exceptionCatcher.isEmpty()); - // test getRegionMetrics from non-partitioned table + // test getRegionLoad from non-partitioned table String non_part_tablegroup = "test_no_part"; batchInsert(10000, non_part_tablegroup); start = System.currentTimeMillis(); - metrics = admin.getRegionMetrics(null, TableName.valueOf(non_part_tablegroup)); + regionLoadMap = admin.getRegionLoad(ServerName.valueOf("localhost,1,1"), TableName.valueOf(non_part_tablegroup)); cost = System.currentTimeMillis() - start; - System.out.println("get region metrics time usage: " + cost + "ms, tablegroup: " + non_part_tablegroup); - assertEquals(3, metrics.size()); + System.out.println("get region load time usage: " + cost + "ms, tablegroup: " + non_part_tablegroup); + assertEquals(3, regionLoadMap.size()); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java index 009cda20..43403229 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java @@ -85,7 +85,6 @@ public void testNew() throws Exception { try { hTable1.getTableDescriptor(); - fail(); } catch (Exception e) { assertTrue(true); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index 3d49b929..bfbc4b87 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -208,15 +208,6 @@ public void testFuzzyRowFilter() throws IOException { Assert.assertArrayEquals("FuzzyRowFilter('abc','101','ddd','010')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } - @Test - public void testColumnValueFilter() throws IOException { - ColumnValueFilter filter = new ColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("q"), - CompareOperator.EQUAL, Bytes.toBytes("v")); - System.out.println(Bytes.toString(HBaseFilterUtils.toParseableByteArray(filter))); - Assert.assertArrayEquals("ColumnValueFilter('cf','q',=,'binary:v')".getBytes(), - HBaseFilterUtils.toParseableByteArray(filter)); - } - @Test public void testMultiRowRangeFilter() throws IOException { List ranges = new ArrayList<>(); From 1e603e548505e56d9743b69a268ffbe46011293d Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Thu, 12 Jun 2025 14:14:43 +0800 Subject: [PATCH 02/15] change fastjson to jackson --- pom.xml | 5 ++ .../execute/AbstractObTableMetaExecutor.java | 17 ++++ .../hbase/execute/ObTableMetaExecutor.java | 17 ++++ .../hbase/util/OHCreateTableExecutor.java | 7 +- .../hbase/util/OHDeleteTableExecutor.java | 24 +++++- .../oceanbase/hbase/util/OHRegionLocator.java | 34 ++++++-- .../hbase/util/OHRegionLocatorExecutor.java | 44 +++++++--- .../oceanbase/hbase/util/OHRegionMetrics.java | 0 .../hbase/util/OHRegionMetricsExecutor.java | 0 .../util/OHTableAccessControlExecutor.java | 27 +++++- .../hbase/util/OHTableDescriptorExecutor.java | 85 +++++++++++-------- .../hbase/util/OHTableExistsExecutor.java | 31 +++++-- 12 files changed, 221 insertions(+), 70 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java diff --git a/pom.xml b/pom.xml index 7797f2cf..ab0b8209 100644 --- a/pom.xml +++ b/pom.xml @@ -198,6 +198,11 @@ 1.2.12 test + + com.fasterxml.jackson.core + jackson-databind + 2.19.0 + diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java index d6d99da1..1c5d51d4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java @@ -1,3 +1,20 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.execute; import com.alipay.oceanbase.rpc.ObTableClient; diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java index 4e8a0ffb..d55ff275 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java @@ -1,3 +1,20 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.execute; import com.alipay.oceanbase.rpc.ObTableClient; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java index a56e8097..4bab8e3b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java @@ -17,7 +17,7 @@ package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -53,7 +53,7 @@ public void createTable(TableDescriptor tableDescriptor, byte[][] splitKeys) thr final ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); - requestData.put("htable_name", tableDescriptor.getTableName().getName()); + requestData.put("htable_name", tableDescriptor.getTableName().getNameAsString()); Map> columnFamilies = new HashMap<>(); for (ColumnFamilyDescriptor columnDescriptor : tableDescriptor.getColumnFamilies()) { Map columnFamily = new HashMap<>(); @@ -62,7 +62,8 @@ public void createTable(TableDescriptor tableDescriptor, byte[][] splitKeys) thr columnFamilies.put(columnDescriptor.getNameAsString(), columnFamily); } requestData.put("column_families", columnFamilies); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); execute(client, request); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java index 47280128..1f689596 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java @@ -1,7 +1,23 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.annotation.JSONField; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -24,7 +40,6 @@ public ObTableRpcMetaType getMetaType() { return ObTableRpcMetaType.HTABLE_DELETE_TABLE; } - @Override public Void parse(ObTableMetaResponse response) throws IOException { // do nothing, error will be thrown from table @@ -36,7 +51,8 @@ public Void deleteTable(String tableName) throws IOException { request.setMetaType(getMetaType()); Map requestDataMap = new HashMap<>(); requestDataMap.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestDataMap); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestDataMap); request.setData(jsonData); return execute(tableClient, request); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java index ab99bc22..4894501c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java @@ -1,3 +1,20 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.rpc.ObTableClient; @@ -13,16 +30,16 @@ import java.util.List; public class OHRegionLocator implements RegionLocator { - private byte[][] startKeys; - private byte[][] endKeys; - private ObTableClient tableClient; - private TableName tableName; + private byte[][] startKeys; + private byte[][] endKeys; + private ObTableClient tableClient; + private TableName tableName; private List regionLocations; public OHRegionLocator(byte[][] startKeys, byte[][] endKeys, - List regionLocations, - TableName tableName, ObTableClient tableClient) { + List regionLocations, TableName tableName, + ObTableClient tableClient) { this.startKeys = startKeys; this.endKeys = endKeys; this.regionLocations = regionLocations; @@ -44,7 +61,8 @@ public HRegionLocation getRegionLocation(byte[] bytes) throws IOException { @Override public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOException { if (b || regionLocations.isEmpty()) { - OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), tableClient); + OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), + tableClient); try { RegionLocator location = executor.getRegionLocator(tableName.toString()); this.startKeys = location.getStartKeys(); @@ -52,7 +70,7 @@ public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOExcep this.regionLocations = location.getAllRegionLocations(); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else { throw e; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java index 435cd6f4..c901d4ad 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java @@ -1,13 +1,29 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.constant.Constants; -import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; -import com.alipay.oceanbase.rpc.location.model.TableEntry; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; @@ -19,7 +35,7 @@ import java.util.stream.IntStream; public class OHRegionLocatorExecutor extends AbstractObTableMetaExecutor { - private final String tableName; + private final String tableName; private final ObTableClient client; OHRegionLocatorExecutor(String tableName, ObTableClient client) { @@ -42,7 +58,8 @@ public ObTableRpcMetaType getMetaType() { public OHRegionLocator parse(ObTableMetaResponse response) throws IOException { try { final String jsonData = response.getData(); - final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(jsonData)) + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(jsonData)) .orElseThrow(() -> new IOException("jsonMap is null")); /* { @@ -77,14 +94,17 @@ public OHRegionLocator parse(ObTableMetaResponse response) throws IOException { ] } */ - - final List partitions = Optional.>ofNullable(jsonMap.getJSONArray("partitions")) + JsonNode partitionsNode = Optional.ofNullable(jsonMap.get("partitions")) .orElseThrow(() -> new IOException("partitions is null")); + List partitions = objectMapper.convertValue(partitionsNode, new TypeReference>(){}); - final List tableIdDict = Optional.>ofNullable(jsonMap.getJSONArray("table_id_dict")) + JsonNode tableIdDictNode = Optional.ofNullable(jsonMap.get("table_id_dict")) .orElseThrow(() -> new IOException("tableIdDict is null")); - final List replicaDict = Optional.>ofNullable(jsonMap.getJSONArray("replica_dict")) + List tableIdDict = objectMapper.convertValue(tableIdDictNode, new TypeReference>(){}); + + JsonNode replicaDictNode = Optional.ofNullable(jsonMap.get("replica_dict")) .orElseThrow(() -> new IOException("replicaDict is null")); + List replicaDict = objectMapper.convertValue(replicaDictNode, new TypeReference>(){}); final boolean isHashLikePartition = partitions.stream() .map(obj -> (List) obj) @@ -213,8 +233,8 @@ public OHRegionLocator getRegionLocator(final String tableName) throws IOExcepti request.setMetaType(getMetaType()); final Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - - final String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); return execute(client, request); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java new file mode 100644 index 00000000..e69de29b diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java new file mode 100644 index 00000000..e69de29b diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java index 090da7e4..9bf0075b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java @@ -1,6 +1,23 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -15,7 +32,7 @@ import java.util.Map; public class OHTableAccessControlExecutor extends AbstractObTableMetaExecutor { - private final ObTableClient tableClient; + private final ObTableClient tableClient; private final ObTableRpcMetaType type; OHTableAccessControlExecutor(ObTableClient tableClient, ObTableRpcMetaType type) { @@ -38,7 +55,8 @@ public void enableTable(String tableName) throws IOException, TableNotFoundExcep request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); execute(tableClient, request); } @@ -48,7 +66,8 @@ public void disableTable(String tableName) throws IOException, TableNotFoundExce request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); execute(tableClient, request); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java index dae981f0..de0ee103 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -1,7 +1,25 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -18,7 +36,7 @@ import java.util.Optional; public class OHTableDescriptorExecutor extends AbstractObTableMetaExecutor { - private final String tableName; + private final String tableName; private final ObTableClient client; public OHTableDescriptorExecutor(String tableName, ObTableClient client) { @@ -30,35 +48,35 @@ public OHTableDescriptorExecutor(String tableName, ObTableClient client) { public HTableDescriptor parse(ObTableMetaResponse response) throws IOException { try { final String jsonData = response.getData(); - final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(jsonData)) + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(jsonData)) .orElseThrow(() -> new IOException("jsonMap is null")); /* { - "cfDesc": { + "cfDescs": { "cf1": { - "TTL":604800 + "TTL":604800 }, "cf2": { - "TTL":259200 + "TTL":259200 } }, "tbDesc": { - "name":"test" + "name":"test", + "state":"disable" ("enable") } } */ HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); - JSONObject cfDesc = jsonMap.getJSONObject("cfDescs"); - if (cfDesc != null) { - for (Map.Entry entry : cfDesc.entrySet()) { - String cfName = entry.getKey(); - JSONObject attributes = (JSONObject) entry.getValue(); - HColumnDescriptor cf = new HColumnDescriptor(cfName); - cf.setTimeToLive(attributes.getIntValue("TTL")); - tableDescriptor.addFamily(cf); - } - } else { - throw new IOException("cfDesc is null"); + JsonNode cfDescsNode = Optional.ofNullable(jsonMap.get("cfDescs")) + .orElseThrow(() -> new IOException("cfDesc is null")); + Map cfDescsMap = objectMapper.convertValue(cfDescsNode, new TypeReference>(){}); + for (Map.Entry entry : cfDescsMap.entrySet()) { + String cfName = entry.getKey(); + JsonNode attributes = (JsonNode) entry.getValue(); + HColumnDescriptor cf = new HColumnDescriptor(cfName); + cf.setTimeToLive(attributes.get("TTL").asInt()); + tableDescriptor.addFamily(cf); } return tableDescriptor; } catch (IllegalArgumentException e) { @@ -71,14 +89,13 @@ public ObTableRpcMetaType getMetaType() throws IOException { return ObTableRpcMetaType.HTABLE_GET_DESC; } - public HTableDescriptor getTableDescriptor() throws IOException { final ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); final Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - - final String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); return execute(client, request); @@ -90,22 +107,21 @@ public boolean isDisable() throws IOException { request.setMetaType(getMetaType()); final Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - - final String jsonData = JSON.toJSONString(requestData); + final ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); try { ObTableMetaResponse response = innerExecute(client, request); final String responseData = response.getData(); - final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(responseData)) + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(responseData)) .orElseThrow(() -> new IOException("jsonMap is null")); - JSONObject tbDesc = jsonMap.getJSONObject("tableDesc"); - if (tbDesc != null) { - String state = tbDesc.getString("state"); - if (state.compareToIgnoreCase("disable") == 0) { - isDisable = true; - } else { - isDisable = false; - } + JsonNode tbDesc = Optional.ofNullable(jsonMap.get("tableDesc")) + .orElseThrow(() -> new IOException("tableDesc is null")); + String state = tbDesc.get("state").asText(); + if (state.compareToIgnoreCase("disable") == 0) { + isDisable = true; + } else { + isDisable = false; } } catch (IOException e) { throw e; @@ -113,7 +129,8 @@ public boolean isDisable() throws IOException { return isDisable; } - private ObTableMetaResponse innerExecute(ObTableClient client, ObTableMetaRequest request) throws IOException { + private ObTableMetaResponse innerExecute(ObTableClient client, ObTableMetaRequest request) + throws IOException { if (request.getMetaType() != getMetaType()) { throw new IOException("Invalid meta type, expected " + getMetaType()); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java index 7b827226..ffaaca18 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java @@ -1,7 +1,24 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -11,6 +28,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; public class OHTableExistsExecutor extends AbstractObTableMetaExecutor { private final ObTableClient tableClient; @@ -27,8 +45,10 @@ public ObTableRpcMetaType getMetaType() throws IOException { @Override public Boolean parse(ObTableMetaResponse response) throws IOException { String jsonData = response.getData(); - JSONObject object = JSONObject.parseObject(jsonData); - return object.getBoolean("exists"); + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = Optional.ofNullable(objectMapper.readTree(jsonData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + return jsonNode.get("exists").asBoolean(); } public Boolean tableExists(String tableName) throws IOException { @@ -36,7 +56,8 @@ public Boolean tableExists(String tableName) throws IOException { request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); return execute(tableClient, request); } From 1d294c8256a710ffd528e8be11bc3401b474796d Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Thu, 12 Jun 2025 11:30:15 +0800 Subject: [PATCH 03/15] [Test] add defensive test for htable ddl --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 54 +++-- .../hbase/OHTableAdminInterfaceTest.java | 201 +++++++++++++++++- 2 files changed, 224 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index ad12c90e..c81668c9 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -174,20 +174,19 @@ public TableDescriptor getDescriptor(TableName tableName) throws IOException { @Override public void createTable(TableDescriptor tableDescriptor) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); -// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); -// OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); -// try { -// executor.createTable(tableDescriptor, null); -// } catch (IOException e) { -// if (e.getCause() instanceof ObTableTransportException -// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { -// throw new TimeoutIOException(e.getCause()); -// } else { -// throw e; -// } -// } + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); + OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); + try { + executor.createTable(tableDescriptor, null); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override @@ -207,20 +206,19 @@ public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] b @Override public void deleteTable(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); -// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); -// OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); -// try { -// executor.deleteTable(tableName.getNameAsString()); -// } catch (IOException e) { -// if (e.getCause() instanceof ObTableTransportException -// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { -// throw new TimeoutIOException(e.getCause()); -// } else { -// throw e; -// } -// } + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); + try { + executor.deleteTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 9e1963cb..6930347f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -34,6 +34,7 @@ import java.io.IOException; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.LinkedList; @@ -780,17 +781,17 @@ public void testCreateDeleteTable() throws Exception { assertTrue(admin.tableExists(tableName)); // TODO: show create table, need to be replace by getDescriptor java.sql.Connection conn = ObHTableTestUtil.getConnection(); - String selectSql = "show create table " + tableName.getNameAsString() + "$" + cf1; + String selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf1); System.out.println("execute sql: " + selectSql); java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); ResultSetPrinter.print(resultSet); - selectSql = "show create table " + tableName.getNameAsString() + "$" + cf2; + selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf2); System.out.println("execute sql: " + selectSql); resultSet = conn.createStatement().executeQuery(selectSql); ResultSetPrinter.print(resultSet); - selectSql = "show create table " + tableName.getNameAsString() + "$" + cf3; + selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf3); System.out.println("execute sql: " + selectSql); resultSet = conn.createStatement().executeQuery(selectSql); ResultSetPrinter.print(resultSet); @@ -1033,4 +1034,198 @@ public void testConcurCreateDelPerf() throws Exception { duration = System.currentTimeMillis() - start; System.out.println("delete " + tableNums + " tables cost " + duration + " ms."); } + + @Test + public void testHTableDDLDefense() throws Exception { + TableName tableName = TableName.valueOf("testHTableDefense"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + // 1. construct htable desc and column family desc + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd1.setMaxVersions(1); + hcd1.setTimeToLive(86400); + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + + // 2. execute create table and check exists + try { + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + admin.createTable(htd); + assertTrue(admin.tableExists(tableName)); + + /// execute the following ddl stmt in created by admin table, should be prohibited + // 4. alter table add constraint + try { + String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 5. alter table add index + try { + String sql = "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 5. alter table add fk + try { + String sql = "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 6. alter table modify column to lob + try { + String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 7. create trigger + try { + String sql = " CREATE TRIGGER hbase_trigger_1" + + " AFTER INSERT ON testHTableDefense$cf1 FOR EACH ROW" + + " BEGIN END"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 8. create view + try { + String sql = " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 9. alter view + try { + String sql = "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 10. create index + try { + String sql = " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + + // 11. explicit create table and specify created_by:admin, should be prohibited + try { + String sql = "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CREATED_BY\": \"ADMIN\"}}'"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 12. alter table to created_by:admin, should be prohibited + try { + String sql1 = "CREATE TABLE testHTableDefense$cf3(a int primary key)"; + System.out.println("execute sql: " + sql1); + conn.createStatement().execute(sql1); + String sql2 = "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CREATED_BY\": \"ADMIN\"}}'"; + System.out.println("execute sql: " + sql2); + conn.createStatement().execute(sql2); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("alter table kv attributes to created by admin not supported", e.getMessage()); + // clean table + String sql3 = "drop table if exists testHTableDefense$cf3"; + System.out.println("execute sql: " + sql3); + conn.createStatement().execute(sql3); + } + + // 13. disable a htable did not created by admin is not suppported + try { + String sql1 = "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2"; + System.out.println("execute sql: " + sql1); + conn.createStatement().execute(sql1); + String sql2 = "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2"; + System.out.println("execute sql: " + sql2); + conn.createStatement().execute(sql2); + admin.disableTable(TableName.valueOf("testHTableDefense2")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + + } + + // 14. delete a htable did not created by admin is not suppported + try { + String sql1 = "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2"; + System.out.println("execute sql: " + sql1); + conn.createStatement().execute(sql1); + String sql2 = "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2"; + System.out.println("execute sql: " + sql2); + conn.createStatement().execute(sql2); + admin.deleteTable(TableName.valueOf("testHTableDefense2")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + admin.disableTable(tableName); + admin.deleteTable(tableName); + String sql1 = "DROP TABLE IF EXISTS testHTableDefense2$cf4"; + System.out.println("execute sql: " + sql1); + conn.createStatement().execute(sql1); + String sql2 = "DROP TABLE IF EXISTS testHTableDefense2$cf5"; + System.out.println("execute sql: " + sql2); + conn.createStatement().execute(sql2); + String sql3 = "DROP TABLEGROUP IF EXISTS testHTableDefense2"; + System.out.println("execute sql: " + sql3); + conn.createStatement().execute(sql3); + } + } } From 21bca0ca34bc4bfc4333f3c573e637f3b0c9af4a Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Thu, 12 Jun 2025 11:53:39 +0800 Subject: [PATCH 04/15] [Chore] open the limiation for other htable admin operation --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index c81668c9..5d65907f 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -248,20 +248,19 @@ public Future truncateTableAsync(TableName tableName, boolean b) throws IO @Override public void enableTable(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); -// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); -// OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); -// try { -// executor.enableTable(tableName.getNameAsString()); -// } catch (IOException e) { -// if (e.getCause() instanceof ObTableTransportException -// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { -// throw new TimeoutIOException(e.getCause()); -// } else { -// throw e; -// } -// } + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); + try { + executor.enableTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override @@ -313,14 +312,12 @@ public HTableDescriptor[] disableTables(Pattern pattern) throws IOException { @Override public boolean isTableEnabled(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// return isDisabled(tableName) == false; + return isDisabled(tableName) == false; } @Override public boolean isTableDisabled(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// return isDisabled(tableName) == true; + return isDisabled(tableName) == true; } private boolean isDisabled(TableName tableName) throws IOException { From a63ccad7725fea595b7e04e98daf03f0b692c23c Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Thu, 12 Jun 2025 14:28:57 +0800 Subject: [PATCH 05/15] [Test] modify test cases after observer changed --- .../hbase/OHTableAdminInterfaceTest.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 6930347f..09048c66 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -1067,10 +1067,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 5. alter table add index @@ -1078,10 +1078,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 5. alter table add fk @@ -1089,10 +1089,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 6. alter table modify column to lob @@ -1100,10 +1100,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 7. create trigger @@ -1113,10 +1113,10 @@ public void testHTableDDLDefense() throws Exception { " BEGIN END"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 8. create view @@ -1124,10 +1124,10 @@ public void testHTableDDLDefense() throws Exception { String sql = " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 9. alter view @@ -1135,10 +1135,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 10. create index @@ -1146,22 +1146,22 @@ public void testHTableDDLDefense() throws Exception { String sql = " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 11. explicit create table and specify created_by:admin, should be prohibited try { - String sql = "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CREATED_BY\": \"ADMIN\"}}'"; + String sql = "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 12. alter table to created_by:admin, should be prohibited @@ -1169,11 +1169,11 @@ public void testHTableDDLDefense() throws Exception { String sql1 = "CREATE TABLE testHTableDefense$cf3(a int primary key)"; System.out.println("execute sql: " + sql1); conn.createStatement().execute(sql1); - String sql2 = "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CREATED_BY\": \"ADMIN\"}}'"; + String sql2 = "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'"; System.out.println("execute sql: " + sql2); conn.createStatement().execute(sql2); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("alter table kv attributes to created by admin not supported", e.getMessage()); // clean table @@ -1191,8 +1191,8 @@ public void testHTableDDLDefense() throws Exception { System.out.println("execute sql: " + sql2); conn.createStatement().execute(sql2); admin.disableTable(TableName.valueOf("testHTableDefense2")); + fail(); } catch (Exception e) { - e.printStackTrace(); Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); } @@ -1206,8 +1206,8 @@ public void testHTableDDLDefense() throws Exception { System.out.println("execute sql: " + sql2); conn.createStatement().execute(sql2); admin.deleteTable(TableName.valueOf("testHTableDefense2")); + fail(); } catch (Exception e) { - e.printStackTrace(); Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); } From a472cecb6aa7f657afc07c9e08a35c185ec61b83 Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Thu, 12 Jun 2025 17:03:11 +0800 Subject: [PATCH 06/15] add htable ddl concurrent test case (#241) * add htable ddl concurrent test case * create table by admin * [Test] add defensive test for htable ddl * [Chore] open the limiation for other htable admin operation * [Test] modify test cases after observer changed * add htable ddl concurrent test case --------- Co-authored-by: shenyunlong.syl --- pom.xml | 2 +- .../hbase/OHTableAdminInterfaceTest.java | 297 +++++++----------- 2 files changed, 120 insertions(+), 179 deletions(-) diff --git a/pom.xml b/pom.xml index ab0b8209..255b54ac 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ ${project.encoding} UTF-8 1.7.21 - 2.0.0 + 2.0.1-SNAPSHOT diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 09048c66..d2d81e22 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -36,10 +36,9 @@ import java.io.IOException; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import java.util.concurrent.*; +import java.util.stream.Collectors; import java.util.Map; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; @@ -270,59 +269,31 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } + public static void createTable(Admin admin, TableName tableName, String... columnFamilies) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + // Add column families + for (String cf : columnFamilies) { + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(cf)); + htd.addFamily(hcd); + } + // Create the table + admin.createTable(htd); + } + @Test public void testAdminEnDisableTable() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); - st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + - "CREATE DATABASE IF NOT EXISTS `n1`;\n" + - "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test` SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `n1:test$family_group` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test`;" + - "\n" + - "CREATE TABLE IF NOT EXISTS `n1:test$family1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test`;"); + st.execute("CREATE DATABASE IF NOT EXISTS `en_dis`"); + st.close(); + conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); - assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + createTable(admin, TableName.valueOf("test_en_dis_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("en_dis", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("en_dis", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); // 1. disable a non-existed table { IOException thrown = assertThrows(IOException.class, @@ -334,28 +305,28 @@ public void testAdminEnDisableTable() throws Exception { } // 2. write an enabled table, should succeed { - if (admin.isTableDisabled(TableName.valueOf("test_multi_cf"))) { - admin.enableTable(TableName.valueOf("test_multi_cf")); + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); } - batchInsert(10, "test_multi_cf"); - batchGet(10, "test_multi_cf"); + batchInsert(10, "test_en_dis_tb"); + batchGet(10, "test_en_dis_tb"); } // 3. disable a enable table { - if (admin.isTableEnabled(TableName.valueOf("test_multi_cf"))) { - admin.disableTable(TableName.valueOf("test_multi_cf")); + if (admin.isTableEnabled(TableName.valueOf("test_en_dis_tb"))) { + admin.disableTable(TableName.valueOf("test_en_dis_tb")); } // write and read disable table, should fail try { - batchInsert(10, "test_multi_cf"); + batchInsert(10, "test_en_dis_tb"); Assert.fail(); } catch (IOException ex) { Assert.assertTrue(ex.getCause() instanceof ObTableException); System.out.println(ex.getCause().getMessage()); } try { - batchGet(10, "test_multi_cf"); + batchGet(10, "test_en_dis_tb"); Assert.fail(); } catch (IOException ex) { Assert.assertTrue(ex.getCause() instanceof ObTableException); @@ -367,48 +338,48 @@ public void testAdminEnDisableTable() throws Exception { // 4. enable a disabled table { - if (admin.isTableDisabled(TableName.valueOf("test_multi_cf"))) { - admin.enableTable(TableName.valueOf("test_multi_cf")); + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); } // write an enabled table, should succeed - batchInsert(10, "test_multi_cf"); - batchGet(10, "test_multi_cf"); + batchInsert(10, "test_en_dis_tb"); + batchGet(10, "test_en_dis_tb"); } // 5. enable an enabled table { - if (admin.isTableDisabled(TableName.valueOf("n1", "test"))) { - admin.enableTable(TableName.valueOf("n1", "test")); + if (admin.isTableDisabled(TableName.valueOf("en_dis", "test"))) { + admin.enableTable(TableName.valueOf("en_dis", "test")); } try { - admin.enableTable(TableName.valueOf("n1", "test")); + admin.enableTable(TableName.valueOf("en_dis", "test")); Assert.fail(); } catch (IOException ex) { Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, ((ObTableException) ex.getCause()).getErrorCode()); } } // 6. disable a disabled table { - if (admin.isTableEnabled(TableName.valueOf("n1", "test"))) { - admin.disableTable(TableName.valueOf("n1", "test")); + if (admin.isTableEnabled(TableName.valueOf("en_dis", "test"))) { + admin.disableTable(TableName.valueOf("en_dis", "test")); } try { - admin.disableTable(TableName.valueOf("n1", "test")); + admin.disableTable(TableName.valueOf("en_dis", "test")); Assert.fail(); } catch (IOException ex) { Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, ((ObTableException) ex.getCause()).getErrorCode()); } } - admin.deleteTable(TableName.valueOf("test_multi_cf")); - assertFalse(admin.tableExists(TableName.valueOf("test_multi_cf"))); - admin.deleteTable(TableName.valueOf("n1", "test")); - assertFalse(admin.tableExists(TableName.valueOf("n1", "test"))); + admin.deleteTable(TableName.valueOf("test_en_dis_tb")); + assertFalse(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); + admin.deleteTable(TableName.valueOf("en_dis", "test")); + assertFalse(admin.tableExists(TableName.valueOf("en_dis", "test"))); } private void batchGet(int rows, String tablegroup) throws Exception { @@ -433,78 +404,60 @@ private void batchGet(int rows, String tablegroup) throws Exception { public void testRegionLoad() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); - st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLEGROUP IF NOT EXISTS test_no_part SHARDING = 'ADAPTIVE';\n"+ - "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group1` (\n" + + st.execute("CREATE TABLEGROUP IF NOT EXISTS test_get_region_metrics SHARDING = 'ADAPTIVE';\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group1` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_no_part;\n" + - "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group2` (\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group2` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_no_part;\n" + - "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group3` (\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group3` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_no_part;\n" + - "CREATE DATABASE IF NOT EXISTS `n1`;\n" + - "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group1` (\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE DATABASE IF NOT EXISTS `get_region`;\n" + + "use `get_region`;\n" + + "CREATE TABLEGROUP IF NOT EXISTS `get_region:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group1` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group2` (\n" + + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group2` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group3` (\n" + + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group3` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); st.close(); conn.close(); - String tablegroup1 = "test_multi_cf"; - String tablegroup2 = "n1:test_multi_cf"; + String tablegroup1 = "test_get_region_metrics"; + String tablegroup2 = "get_region:test_multi_cf"; Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); @@ -613,79 +566,33 @@ public void testRegionLoad() throws Exception { public void testAdminDeleteTable() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); - st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE DATABASE IF NOT EXISTS `n1`;\n" + - "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); + st.execute("CREATE DATABASE IF NOT EXISTS `del_tb`"); st.close(); conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - assertTrue(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); - assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + createTable(admin, TableName.valueOf("test_del_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("del_tb", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("del_tb", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_del_tb"))); IOException thrown = assertThrows(IOException.class, () -> { admin.deleteTable(TableName.valueOf("tablegroup_not_exists")); }); Assert.assertTrue(thrown.getCause() instanceof ObTableException); Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); - admin.deleteTable(TableName.valueOf("n1", "test_multi_cf")); - admin.deleteTable(TableName.valueOf("test_multi_cf")); - assertFalse(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); - assertFalse(admin.tableExists(TableName.valueOf("test_multi_cf"))); + admin.deleteTable(TableName.valueOf("del_tb", "test")); + admin.deleteTable(TableName.valueOf("test_del_tb")); + assertFalse(admin.tableExists(TableName.valueOf("del_tb", "test"))); + assertFalse(admin.tableExists(TableName.valueOf("test_del_tb"))); } @Test public void testAdminTableExists() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); - st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE DATABASE IF NOT EXISTS `n1`;\n" + - "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';"); + st.execute("CREATE DATABASE IF NOT EXISTS `exist_tb`"); st.close(); conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); @@ -697,14 +604,16 @@ public void testAdminTableExists() throws Exception { TableName.valueOf("random_string$"); }); Assert.assertFalse(admin.tableExists(TableName.valueOf("tablegroup_not_exists"))); - Assert.assertTrue(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); - Assert.assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + createTable(admin, TableName.valueOf("test_exist_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("exist_tb", "test"), "cf1", "cf2", "cf3");å + Assert.assertTrue(admin.tableExists(TableName.valueOf("test_exist_tb"))); + Assert.assertTrue(admin.tableExists(TableName.valueOf("exist_tb", "test"))); } private void batchInsert(int rows, String tablegroup) throws Exception { - byte[] family1 = Bytes.toBytes("family_with_group1"); - byte[] family2 = Bytes.toBytes("family_with_group2"); - byte[] family3 = Bytes.toBytes("family_with_group3"); + byte[] family1 = Bytes.toBytes("cf1"); + byte[] family2 = Bytes.toBytes("cf2"); + byte[] family3 = Bytes.toBytes("cf3"); byte[] family1_column1 = "family1_column1".getBytes(); byte[] family1_column2 = "family1_column2".getBytes(); byte[] family1_column3 = "family1_column3".getBytes(); @@ -840,6 +749,7 @@ void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreE hcd1.setMaxVersions(1); hcd1.setTimeToLive(86400); HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + List originalColumnDescriptors = Arrays.asList(hcd1, hcd2, hcd3); // 1. generate create table task, one task per table List> tasks = new ArrayList<>(); @@ -847,11 +757,19 @@ void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreE int finalI = i; tasks.add(()->{ HTableDescriptor htd = new HTableDescriptor(tableNames.get(finalI)); - htd.addFamily(hcd1); - htd.addFamily(hcd2); - htd.addFamily(hcd3); + List columnDescriptors = new ArrayList<>(originalColumnDescriptors); + // 随机打乱列族顺序 + Collections.shuffle(columnDescriptors); + String shuffledCfNames = columnDescriptors.stream() + .map(hcd -> Bytes.toString(hcd.getName())) + .collect(Collectors.joining(", ")); + System.out.println("Table " + tableNames.get(finalI) + " shuffled column families: " + shuffledCfNames); + for (HColumnDescriptor hcd : columnDescriptors) { + htd.addFamily(hcd); + } try { admin.createTable(htd); + System.out.println("success to create table:" + tableNames.get(finalI)); } catch (Exception e) { System.out.println(e); if (!ignoreException) { @@ -892,9 +810,31 @@ void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreE } // 4. disable all tables; + List> disableTasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + disableTasks.add(()->{ + try { + admin.disableTable(tableNames.get(finalI)); + System.out.println("success to disable table:" + tableNames.get(finalI)); + } catch (Exception e) { + System.out.println(e); + if (!ignoreException) { + throw e; + } + } + return null; + }); + } + ExecutorService disExecutorService = Executors.newFixedThreadPool(tableNums); + disExecutorService.invokeAll(disableTasks); + disExecutorService.shutdown(); + disExecutorService.awaitTermination(1, TimeUnit.MINUTES); + + assertTrue(admin.isTableDisabled(tableNames.get(0))); for (int i = 0; i < tableNames.size(); i++) { TableName tableName = tableNames.get(i); - admin.disableTable(tableName); + assertTrue(admin.isTableDisabled(tableName)); } // 5. generate delete table task @@ -904,6 +844,7 @@ void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreE delTasks.add(()->{ try { admin.deleteTable(tableNames.get(finalI)); + System.out.println("success to drop table:" + tableNames.get(finalI)); } catch (Exception e) { System.out.println(e); if (!ignoreException) { From 966dbca5c880558e21517fc2f3e848dd766484b0 Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Thu, 12 Jun 2025 17:09:27 +0800 Subject: [PATCH 07/15] fix for compile --- .../com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index d2d81e22..2d7004f3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -605,7 +605,7 @@ public void testAdminTableExists() throws Exception { }); Assert.assertFalse(admin.tableExists(TableName.valueOf("tablegroup_not_exists"))); createTable(admin, TableName.valueOf("test_exist_tb"), "cf1", "cf2", "cf3"); - createTable(admin, TableName.valueOf("exist_tb", "test"), "cf1", "cf2", "cf3");å + createTable(admin, TableName.valueOf("exist_tb", "test"), "cf1", "cf2", "cf3"); Assert.assertTrue(admin.tableExists(TableName.valueOf("test_exist_tb"))); Assert.assertTrue(admin.tableExists(TableName.valueOf("exist_tb", "test"))); } From 286f12ef94f17f10bd9cbd945c9637c57c9e87a0 Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Thu, 12 Jun 2025 17:26:12 +0800 Subject: [PATCH 08/15] Merge pull request #243 from oceanbase/rm_fastjson_dep Change fastjson to jackson --- .../java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java | 0 .../com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java delete mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java deleted file mode 100644 index e69de29b..00000000 diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java deleted file mode 100644 index e69de29b..00000000 From bca31fdfa31f16c6b218de23953ee145e7bdc2f5 Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Fri, 13 Jun 2025 11:18:57 +0800 Subject: [PATCH 09/15] [Test] add testcases for pk and kv_attributes --- .../hbase/OHTableAdminInterfaceTest.java | 329 ++++++++++-------- .../hbase/util/ObHTableTestUtil.java | 5 + 2 files changed, 190 insertions(+), 144 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 2d7004f3..d168c5a1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -42,6 +42,7 @@ import java.util.Map; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.executeSQL; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; import static org.junit.Assert.assertFalse; @@ -290,96 +291,125 @@ public void testAdminEnDisableTable() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - createTable(admin, TableName.valueOf("test_en_dis_tb"), "cf1", "cf2", "cf3"); - createTable(admin, TableName.valueOf("en_dis", "test"), "cf1", "cf2", "cf3"); - assertTrue(admin.tableExists(TableName.valueOf("en_dis", "test"))); - assertTrue(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); - // 1. disable a non-existed table - { - IOException thrown = assertThrows(IOException.class, - () -> { - admin.disableTable(TableName.valueOf("tablegroup_not_exists")); - }); - assertTrue(thrown.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); - } - // 2. write an enabled table, should succeed - { - if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { - admin.enableTable(TableName.valueOf("test_en_dis_tb")); + try { + createTable(admin, TableName.valueOf("test_en_dis_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("en_dis", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("en_dis", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); + String kvAttrStrDefault = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\"}}"; + String kvAttributeDisable = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"disable\"}}"; + String kvAttributeEnable = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"enable\"}}"; + + checkKVAttributes("test_en_dis_tb$cf1", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf2", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf3", kvAttrStrDefault); + // 1. disable a non-existed table + { + IOException thrown = assertThrows(IOException.class, + () -> { + admin.disableTable(TableName.valueOf("tablegroup_not_exists")); + }); + assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); } - batchInsert(10, "test_en_dis_tb"); - batchGet(10, "test_en_dis_tb"); - } + // 2. write an enabled table, should succeed + { + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf2", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf3", kvAttrStrDefault); - // 3. disable a enable table - { - if (admin.isTableEnabled(TableName.valueOf("test_en_dis_tb"))) { - admin.disableTable(TableName.valueOf("test_en_dis_tb")); - } - // write and read disable table, should fail - try { batchInsert(10, "test_en_dis_tb"); - Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - System.out.println(ex.getCause().getMessage()); - } - try { batchGet(10, "test_en_dis_tb"); - Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, - ((ObTableException) ex.getCause()).getErrorCode()); } - } + // 3. disable a enable table + { + if (admin.isTableEnabled(TableName.valueOf("test_en_dis_tb"))) { + admin.disableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttributeDisable); + checkKVAttributes("test_en_dis_tb$cf2", kvAttributeDisable); + checkKVAttributes("test_en_dis_tb$cf3", kvAttributeDisable); + // write and read disable table, should fail + try { + batchInsert(10, "test_en_dis_tb"); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + System.out.println(ex.getCause().getMessage()); + } + try { + batchGet(10, "test_en_dis_tb"); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } - // 4. enable a disabled table - { - if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { - admin.enableTable(TableName.valueOf("test_en_dis_tb")); } - // write an enabled table, should succeed - batchInsert(10, "test_en_dis_tb"); - batchGet(10, "test_en_dis_tb"); - } - // 5. enable an enabled table - { - if (admin.isTableDisabled(TableName.valueOf("en_dis", "test"))) { - admin.enableTable(TableName.valueOf("en_dis", "test")); - } - try { - admin.enableTable(TableName.valueOf("en_dis", "test")); - Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, - ((ObTableException) ex.getCause()).getErrorCode()); + // 4. enable a disabled table + { + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttributeEnable); + checkKVAttributes("test_en_dis_tb$cf2", kvAttributeEnable); + checkKVAttributes("test_en_dis_tb$cf3", kvAttributeEnable); + // write an enabled table, should succeed + batchInsert(10, "test_en_dis_tb"); + batchGet(10, "test_en_dis_tb"); } - } - // 6. disable a disabled table - { - if (admin.isTableEnabled(TableName.valueOf("en_dis", "test"))) { - admin.disableTable(TableName.valueOf("en_dis", "test")); + // 5. enable an enabled table + { + if (admin.isTableDisabled(TableName.valueOf("en_dis", "test"))) { + admin.enableTable(TableName.valueOf("en_dis", "test")); + } + checkKVAttributes("en_dis:test$cf1", kvAttrStrDefault); + checkKVAttributes("en_dis:test$cf2", kvAttrStrDefault); + checkKVAttributes("en_dis:test$cf3", kvAttrStrDefault); + try { + admin.enableTable(TableName.valueOf("en_dis", "test")); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } } - try { - admin.disableTable(TableName.valueOf("en_dis", "test")); - Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, - ((ObTableException) ex.getCause()).getErrorCode()); + + // 6. disable a disabled table + { + if (admin.isTableEnabled(TableName.valueOf("en_dis", "test"))) { + admin.disableTable(TableName.valueOf("en_dis", "test")); + } + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + try { + admin.disableTable(TableName.valueOf("en_dis", "test")); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } } - } - admin.deleteTable(TableName.valueOf("test_en_dis_tb")); - assertFalse(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); - admin.deleteTable(TableName.valueOf("en_dis", "test")); - assertFalse(admin.tableExists(TableName.valueOf("en_dis", "test"))); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + admin.deleteTable(TableName.valueOf("test_en_dis_tb")); + assertFalse(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); + admin.deleteTable(TableName.valueOf("en_dis", "test")); + assertFalse(admin.tableExists(TableName.valueOf("en_dis", "test"))); + } } private void batchGet(int rows, String tablegroup) throws Exception { @@ -1003,90 +1033,109 @@ public void testHTableDDLDefense() throws Exception { assertTrue(admin.tableExists(tableName)); /// execute the following ddl stmt in created by admin table, should be prohibited - // 4. alter table add constraint + // 3. alter table add constraint try { - String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 5. alter table add index + // 4. alter table add index try { - String sql = "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 5. alter table add fk + // 5. alter table modify column to lob try { - String sql = "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 6. alter table modify column to lob + // 6. alter hbase admin table add fk try { - String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 7. create trigger + // 7. create a normal table to refer to hbase admin table try { - String sql = " CREATE TRIGGER hbase_trigger_1" + + executeSQL(conn, "create table testHTableDefense_t1(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K));" , true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 8. alter a normal table to refer to hbase admin table + try { + executeSQL(conn, "create table testHTableDefense_t2(a varbinary(1024) primary key)", true); + executeSQL(conn, "alter table testHTableDefense_t2 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K);", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + // 9. create a normal table A to refer to a table mock parent table B, and create table B using hbase admin + try { + executeSQL(conn, "SET foreign_key_checks = 0", true); + + executeSQL(conn, "create table testHTableDefense_t3(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense2$cf1(K));", true); + HTableDescriptor htd2 = new HTableDescriptor(TableName.valueOf("testHTableDefense2")); + HColumnDescriptor hcd4 = new HColumnDescriptor("cf1".getBytes()); + hcd4.setMaxVersions(2); + hcd4.setTimeToLive(172800); + htd2.addFamily(hcd4); + admin.createTable(htd2); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + } + + + // 10. create trigger + try { + executeSQL(conn, " CREATE TRIGGER hbase_trigger_1" + " AFTER INSERT ON testHTableDefense$cf1 FOR EACH ROW" + - " BEGIN END"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + " BEGIN END", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 8. create view + // 11. create view try { - String sql = " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 9. alter view + // 12. alter view try { - String sql = "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 10. create index + // 13. create index try { - String sql = " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); @@ -1094,25 +1143,19 @@ public void testHTableDDLDefense() throws Exception { } - // 11. explicit create table and specify created_by:admin, should be prohibited + // 14. explicit create table and specify created_by:admin, should be prohibited try { - String sql = "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 12. alter table to created_by:admin, should be prohibited + // 15. alter table to created_by:admin, should be prohibited try { - String sql1 = "CREATE TABLE testHTableDefense$cf3(a int primary key)"; - System.out.println("execute sql: " + sql1); - conn.createStatement().execute(sql1); - String sql2 = "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'"; - System.out.println("execute sql: " + sql2); - conn.createStatement().execute(sql2); + executeSQL(conn, "CREATE TABLE testHTableDefense$cf3(a int primary key)", true); + executeSQL(conn, "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); @@ -1123,29 +1166,21 @@ public void testHTableDDLDefense() throws Exception { conn.createStatement().execute(sql3); } - // 13. disable a htable did not created by admin is not suppported + // 16. disable a htable did not created by admin is not suppported try { - String sql1 = "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2"; - System.out.println("execute sql: " + sql1); - conn.createStatement().execute(sql1); - String sql2 = "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2"; - System.out.println("execute sql: " + sql2); - conn.createStatement().execute(sql2); + executeSQL(conn, "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2", true); + executeSQL(conn, "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", true); admin.disableTable(TableName.valueOf("testHTableDefense2")); fail(); } catch (Exception e) { Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); - } - // 14. delete a htable did not created by admin is not suppported + // 17. delete a htable did not created by admin is not suppported try { - String sql1 = "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2"; - System.out.println("execute sql: " + sql1); - conn.createStatement().execute(sql1); - String sql2 = "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2"; - System.out.println("execute sql: " + sql2); - conn.createStatement().execute(sql2); + executeSQL(conn, "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2", true); + executeSQL(conn, + "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", true); admin.deleteTable(TableName.valueOf("testHTableDefense2")); fail(); } catch (Exception e) { @@ -1158,15 +1193,21 @@ public void testHTableDDLDefense() throws Exception { } finally { admin.disableTable(tableName); admin.deleteTable(tableName); - String sql1 = "DROP TABLE IF EXISTS testHTableDefense2$cf4"; - System.out.println("execute sql: " + sql1); - conn.createStatement().execute(sql1); - String sql2 = "DROP TABLE IF EXISTS testHTableDefense2$cf5"; - System.out.println("execute sql: " + sql2); - conn.createStatement().execute(sql2); - String sql3 = "DROP TABLEGROUP IF EXISTS testHTableDefense2"; - System.out.println("execute sql: " + sql3); - conn.createStatement().execute(sql3); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense2$cf4", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense2$cf5", true); + executeSQL(conn, "DROP TABLEGROUP IF EXISTS testHTableDefense2", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t1", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t2", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t3", true); } } + + void checkKVAttributes(String tableName, String kvAttributes) throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery("select kv_attributes from oceanbase.__all_table where table_name = '" + tableName + "'"); + resultSet.next(); + String value = resultSet.getString(1); + Assert.assertEquals(kvAttributes, value); + Assert.assertFalse(resultSet.next()); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index b2c94e6c..b0310567 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -229,4 +229,9 @@ public static boolean secureCompare(byte[] a, byte[] b) { } return diff == 0; } + + public static void executeSQL(Connection conn, String sql, boolean printSQL) throws SQLException { + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } } \ No newline at end of file From dfffe575d1f8cd638988e4c5e5d0b336b502819b Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 13 Jun 2025 17:53:47 +0800 Subject: [PATCH 10/15] fix namespace not exsit --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index 5d65907f..11859135 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -3,9 +3,11 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTableTransportException; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; import org.apache.commons.cli.Option; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; @@ -63,10 +65,27 @@ public Connection getConnection() { @Override public boolean tableExists(TableName tableName) throws IOException { - OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHTableExistsExecutor executor = new OHTableExistsExecutor(tableClient); - return executor.tableExists(tableName.getNameAsString()); + try { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableExistsExecutor executor = new OHTableExistsExecutor(tableClient); + return executor.tableExists(tableName.getNameAsString()); + } catch (Exception e) { + // try to get the original cause + Throwable cause = e.getCause(); + while(cause != null && cause.getCause() != null) { + cause = cause.getCause(); + } + if (cause instanceof ObTableException) { + int errCode = ((ObTableException) cause).getErrorCode(); + // if the original cause is database_not_exist, means namespace in tableName does not exist + // for HBase, namespace not exist will not throw exceptions but will return false + if (errCode == ResultCodes.OB_ERR_BAD_DATABASE.errorCode) { + return false; + } + } + throw e; + } } @Override From 4f8ecd8617a8f9d69210b5be94749d6e7dd18607 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 16 Jun 2025 19:43:56 +0800 Subject: [PATCH 11/15] patch bufferedMutator deals exception --- .../hbase/util/OHBufferedMutatorImpl.java | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index d4ca4aef..d8fdda02 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -232,15 +232,11 @@ private void execute(boolean flushAll) throws IOException { // if commit all successfully, clean execBuffer execBuffer.clear(); } catch (Exception ex) { - LOGGER.error(LCD.convert("01-00026"), ex); - if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { - LOGGER.error(tableName + ": One or more of the operations have failed after retries."); - RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); - // recollect failed mutations - execBuffer.clear(); - for (int i = 0; i < retryException.getNumExceptions(); ++i) { - execBuffer.add((Mutation) retryException.getRow(i)); - } + // do not recollect error operations, notify outside + LOGGER.error("error happens, table name: {}", tableName.getNameAsString(), ex); + if (ex instanceof RetriesExhaustedWithDetailsException) { + LOGGER.error("TableName: {}, One or more of the operations have failed after retries.", tableName.getNameAsString(), ex); + RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex; if (listener != null) { listener.onException(retryException, this); } else { @@ -250,13 +246,6 @@ private void execute(boolean flushAll) throws IOException { LOGGER.error("Errors unrelated to operations occur during mutation operation", ex); throw ex; } - } finally { - for (Mutation mutation : execBuffer) { - long size = mutation.heapSize(); - currentAsyncBufferSize.addAndGet(size); - asyncWriteBuffer.add(mutation); - undealtMutationCount.incrementAndGet(); - } } } From 6f2c8ecc8b362bdaa5e53f752475107d8a2ca4f0 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 16 Jun 2025 19:37:22 +0800 Subject: [PATCH 12/15] fix jackson bug --- .../hbase/util/OHTableDescriptorExecutor.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java index de0ee103..dcf42681 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -17,7 +17,6 @@ package com.alipay.oceanbase.hbase.util; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; @@ -32,8 +31,10 @@ import java.io.IOException; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Optional; +import java.util.stream.Stream; public class OHTableDescriptorExecutor extends AbstractObTableMetaExecutor { private final String tableName; @@ -56,9 +57,11 @@ public HTableDescriptor parse(ObTableMetaResponse response) throws IOException { "cfDescs": { "cf1": { "TTL":604800 + "maxVersions": 3 }, "cf2": { "TTL":259200 + "maxVersions": 2 } }, "tbDesc": { @@ -70,14 +73,17 @@ public HTableDescriptor parse(ObTableMetaResponse response) throws IOException { HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); JsonNode cfDescsNode = Optional.ofNullable(jsonMap.get("cfDescs")) .orElseThrow(() -> new IOException("cfDesc is null")); - Map cfDescsMap = objectMapper.convertValue(cfDescsNode, new TypeReference>(){}); - for (Map.Entry entry : cfDescsMap.entrySet()) { + Stream> stream = cfDescsNode.propertyStream(); + stream.forEach(entry -> { String cfName = entry.getKey(); - JsonNode attributes = (JsonNode) entry.getValue(); + JsonNode value = entry.getValue(); + int ttl = value.path("TTL").asInt(); + int maxVersions = value.path("maxVersions").asInt(); HColumnDescriptor cf = new HColumnDescriptor(cfName); - cf.setTimeToLive(attributes.get("TTL").asInt()); + cf.setTimeToLive(ttl); + cf.setMaxVersions(maxVersions); tableDescriptor.addFamily(cf); - } + }); return tableDescriptor; } catch (IllegalArgumentException e) { throw new IOException("Failed to parse response", e); From 872e4c5f18f6a72e2312adba6d08364de6651b59 Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Tue, 17 Jun 2025 10:02:53 +0800 Subject: [PATCH 13/15] Merge pull request #248 from oceanbase/fix_desc_maxVersions Fix Jackson deserialization bug From 07b33467b3904c48b985ac1236253df8c7884f9a Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 11 Jun 2025 17:03:18 +0800 Subject: [PATCH 14/15] add test case for admin interface error injection --- .../hbase/OHTableAdminInterfaceTest.java | 120 ++++++++++++++++++ .../hbase/util/ObHTableTestUtil.java | 56 +++++++- 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index d168c5a1..7f1de206 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -68,6 +68,58 @@ public OHTablePool setUpPool() throws IOException { return ohTablePool; } + enum ErrSimPoint { + EN_CREATE_HTABLE_TG_FINISH_ERR(2621), + EN_CREATE_HTABLE_CF_FINISH_ERR(2622), + EN_DISABLE_HTABLE_CF_FINISH_ERR(2623), + EN_DELETE_HTABLE_CF_FINISH_ERR(2624); + + private final int errCode; + + ErrSimPoint(int errCode) { + this.errCode = errCode; + } + + public int getErrCode() { + return errCode; + } + } + + private void setErrSimPoint(ErrSimPoint errSimPoint, boolean enable) { + java.sql.Connection connection = null; + java.sql.Statement statement = null; + + try { + connection = ObHTableTestUtil.getSysTenantConnection(); + statement = connection.createStatement(); + + String sql = String.format( + "alter system set_tp tp_no = %d, error_code = 4016, frequency = %d", + errSimPoint.getErrCode(), + enable ? 1 : 0 + ); + + statement.execute(sql); + } catch (Exception e) { + throw new RuntimeException("Error injection setup failed", e); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (Exception e) { + // ignore + } + } + if (connection != null) { + try { + connection.close(); + } catch (Exception e) { + // ignore + } + } + } + } + @Test public void testGetStartEndKeysOHTableClientRange() throws Exception { // Init OHTableClient @@ -1210,4 +1262,72 @@ void checkKVAttributes(String tableName, String kvAttributes) throws Exception { Assert.assertEquals(kvAttributes, value); Assert.assertFalse(resultSet.next()); } + + // NOTE: observer should build with `-DOB_ERRSIM=ON` option, otherwise the test will fail + // This test verifies error injection scenarios for table operations + @Test + public void testCreateTableInjectError() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + byte[] tableName = Bytes.toBytes("test_create_table_inject_error"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + + // 1. open err EN_CREATE_HTABLE_TG_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); + assertFalse("Table should not exist after TG error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, false); + + // 2. open err EN_CREATE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); + assertFalse("Table should not exist after CF error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, false); + + // 3. create table without error + admin.createTable(htd); + assertTrue("Table should exist after normal creation", + admin.tableExists(TableName.valueOf(tableName))); + assertEquals("Table should have 3 column families", 3, + admin.getTableDescriptor(TableName.valueOf(tableName)).getFamilies().size()); + + // 4. open err EN_DISABLE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, true); + admin.disableTable(TableName.valueOf(tableName)); + assertFalse("Table should not be disabled after disable error injection", + admin.isTableDisabled(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, false); + + // 5. disable table without error + admin.disableTable(TableName.valueOf(tableName)); + assertTrue("Table should be disabled after normal disable", + admin.isTableDisabled(TableName.valueOf(tableName))); + + // 6. open err EN_DELETE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, true); + admin.deleteTable(TableName.valueOf(tableName)); + assertTrue("Table should still exist after delete error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, false); + + // 7. delete table without error + admin.deleteTable(TableName.valueOf(tableName)); + assertFalse("Table should not exist after normal delete", + admin.tableExists(TableName.valueOf(tableName))); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index b0310567..ef6cf8e3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -55,7 +55,13 @@ public class ObHTableTestUtil { + "oceanbase?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String SYS_TENANT_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + + "oceanbase?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String SYS_TENANT_USER_NAME = "root@sys"; + public static String SYS_TENANT_PASSWORD = ""; public static String SQL_FORMAT = "truncate %s"; public static List tableNameList = new LinkedList(); public static Connection conn; @@ -172,6 +178,17 @@ static public Connection getSysConnection() { } } + static public Connection getSysTenantConnection() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + Connection conn = DriverManager + .getConnection(SYS_TENANT_JDBC_URL, SYS_TENANT_USER_NAME, SYS_TENANT_PASSWORD); + return conn; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @FunctionalInterface public interface CheckedConsumer { void accept(T t) throws Throwable; @@ -234,4 +251,41 @@ public static void executeSQL(Connection conn, String sql, boolean printSQL) thr System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); } -} \ No newline at end of file + + @FunctionalInterface + public interface CheckedRunnable { + void run() throws Exception; + } + + public static void executeIgnoreUnexpectedError(CheckedRunnable operation) throws Exception { + executeIgnoreExpectedErrors(operation, "OB_ERR_UNEXPECTED"); + } + + public static void executeIgnoreExpectedErrors(CheckedRunnable operation, String... expectedErrorMessages) throws Exception { + try { + operation.run(); + } catch (Exception e) { + boolean shouldIgnore = false; + String[] messagesToCheck = { + e.getMessage(), + e.getCause() != null ? e.getCause().getMessage() : null + }; + + for (String expectedMessage : expectedErrorMessages) { + for (String actualMessage : messagesToCheck) { + if (actualMessage != null && actualMessage.contains(expectedMessage)) { + shouldIgnore = true; + break; + } + } + if (shouldIgnore) { + break; + } + } + + if (!shouldIgnore) { + throw e; + } + } + } +} From 917db23d97b929abab3d71a69c41b3ebd4a867da Mon Sep 17 00:00:00 2001 From: maochongxin Date: Tue, 17 Jun 2025 11:35:53 +0800 Subject: [PATCH 15/15] merge hbase_compat_3_2.0 --- .../com/alipay/oceanbase/hbase/OHTable.java | 44 ++-- .../alipay/oceanbase/hbase/OHTableClient.java | 9 +- .../alipay/oceanbase/hbase/OHTablePool.java | 73 +----- .../hbase/constants/OHConstants.java | 2 +- .../execute/AbstractObTableMetaExecutor.java | 5 +- .../hbase/execute/ObTableMetaExecutor.java | 16 -- .../alipay/oceanbase/hbase/util/OHAdmin.java | 235 ++++++++++++------ .../hbase/util/OHBufferedMutatorImpl.java | 6 +- .../hbase/util/OHConnectionConfiguration.java | 2 +- .../hbase/util/OHConnectionImpl.java | 6 +- .../hbase/util/OHRegionLoadExecutor.java | 64 +++-- .../hbase/util/OHTableDescriptorExecutor.java | 5 +- .../hbase/util/ObTableClientManager.java | 13 +- .../oceanbase/hbase/HTableTestBase.java | 81 +++--- .../oceanbase/hbase/OHConnectionTest.java | 8 +- .../hbase/OHTableAdminInterfaceTest.java | 165 +++++++----- .../oceanbase/hbase/OHTableClientTest.java | 1 - .../OHTableSecondaryPartDeleteTest.java | 2 +- .../OHTableSecondaryPartTTLTest.java | 17 ++ .../hbase/util/ObHTableSecondaryPartUtil.java | 10 +- .../hbase/util/ObHTableTestUtil.java | 79 +++--- .../hbase/util/TableTemplateManager.java | 148 +++++------ 22 files changed, 548 insertions(+), 443 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 7253b23c..47a170f0 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -154,7 +154,7 @@ public class OHTable implements Table { /** * the buffer of put request */ - private final ArrayList writeBuffer = new ArrayList(); + private final ArrayList writeBuffer = new ArrayList(); /** * when the put request reach the write buffer size the do put will * flush commits automatically @@ -459,8 +459,8 @@ private void finishSetUp() { } public static OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, - OHConnectionConfiguration ohConnectionConf) - throws IllegalArgumentException { + OHConnectionConfiguration ohConnectionConf) + throws IllegalArgumentException { if (tableNameString.indexOf(':') != -1) { String[] params = tableNameString.split(":"); if (params.length != 2) { @@ -500,13 +500,15 @@ public Configuration getConfiguration() { @Override public HTableDescriptor getTableDescriptor() throws IOException { - OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableNameString, obTableClient); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableNameString, + obTableClient); return executor.getTableDescriptor(); } @Override public TableDescriptor getDescriptor() throws IOException { - OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableNameString, obTableClient); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableNameString, + obTableClient); return executor.getTableDescriptor(); } @@ -909,10 +911,12 @@ private void processColumnFilters(NavigableSet columnFilters, byte[] family = entry.getKey(); if (entry.getValue() != null) { for (byte[] columnName : entry.getValue()) { - byte[] newQualifier = new byte[family.length + 1/* length of "." */ + columnName.length]; + byte[] newQualifier = new byte[family.length + 1/* length of "." */ + + columnName.length]; System.arraycopy(family, 0, newQualifier, 0, family.length); newQualifier[family.length] = 0x2E; // 0x2E in utf-8 is "." - System.arraycopy(columnName, 0, newQualifier, family.length + 1, columnName.length); + System.arraycopy(columnName, 0, newQualifier, family.length + 1, + columnName.length); columnFilters.add(newQualifier); } } else { @@ -1750,6 +1754,7 @@ public int getReadRpcTimeout() { public void setReadRpcTimeout(int readRpcTimeout) { this.readRpcTimeout = readRpcTimeout; } + @Override public long getReadRpcTimeout(TimeUnit unit) { return getReadRpcTimeout(); @@ -1770,7 +1775,6 @@ public void setWriteRpcTimeout(int writeRpcTimeout) { this.writeRpcTimeout = writeRpcTimeout; } - public void setRuntimeBatchExecutor(ExecutorService runtimeBatchExecutor) { this.obTableClient.setRuntimeBatchExecutor(runtimeBatchExecutor); } @@ -1902,7 +1906,7 @@ private ObHTableFilter buildObHTableFilter(byte[] filterString, TimeRange timeRa if (columnQualifier == null) { obHTableFilter.addSelectColumnQualifier(new byte[0]); } else { - obHTableFilter.addSelectColumnQualifier(columnQualifier); + obHTableFilter.addSelectColumnQualifier(columnQualifier); } } } @@ -1960,11 +1964,11 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { filter.setOffsetPerRowPerCf(scan.getRowOffsetPerColumnFamily()); } if (scan.isReversed()) { - obTableQuery = buildObTableQuery(filter, scan.getStopRow(), scan.includeStopRow(), scan.getStartRow(), - scan.includeStartRow(), true, ts); + obTableQuery = buildObTableQuery(filter, scan.getStopRow(), scan.includeStopRow(), + scan.getStartRow(), scan.includeStartRow(), true, ts); } else { - obTableQuery = buildObTableQuery(filter, scan.getStartRow(), scan.includeStartRow(), scan.getStopRow(), - scan.includeStopRow(), false, ts); + obTableQuery = buildObTableQuery(filter, scan.getStartRow(), scan.includeStartRow(), + scan.getStopRow(), scan.includeStopRow(), false, ts); } obTableQuery.setBatchSize(scan.getBatch()); obTableQuery.setLimit(scan.getLimit()); @@ -2044,10 +2048,11 @@ private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv, KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getTypeByte()); com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, operationType, isTableGroup, family, TTL); - if(isTableGroup) { + if (isTableGroup) { // construct new_kv otherwise filter will fail to match targeted columns byte[] oldQualifier = CellUtil.cloneQualifier(kv); - byte[] newQualifier = new byte[family.length + 1/* length of "." */ + oldQualifier.length]; + byte[] newQualifier = new byte[family.length + 1/* length of "." */ + + oldQualifier.length]; System.arraycopy(family, 0, newQualifier, 0, family.length); newQualifier[family.length] = 0x2E; // 0x2E in utf-8 is "." System.arraycopy(oldQualifier, 0, newQualifier, family.length + 1, oldQualifier.length); @@ -2130,12 +2135,10 @@ private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv, range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), ObObj.getMax())); if (!isTableGroup) { - filter = buildObHTableFilter(null, - new TimeRange(0, kv.getTimestamp() + 1), + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE); } else { - filter = buildObHTableFilter(null, - new TimeRange(0, kv.getTimestamp() + 1), + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE, CellUtil.cloneQualifier(kv)); } } @@ -2155,7 +2158,8 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, Cell newCell = kv; if (isTableGroup && family != null) { byte[] oldQualifier = CellUtil.cloneQualifier(kv); - byte[] newQualifier = new byte[family.length + 1/* length of "." */ + oldQualifier.length]; + byte[] newQualifier = new byte[family.length + 1/* length of "." */ + + oldQualifier.length]; System.arraycopy(family, 0, newQualifier, 0, family.length); newQualifier[family.length] = 0x2E; // 0x2E in utf-8 is "." System.arraycopy(oldQualifier, 0, newQualifier, family.length + 1, oldQualifier.length); diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java index 89c7d35d..122b7756 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java @@ -178,7 +178,8 @@ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, } @Override - public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, RowMutations mutation) throws IOException { + public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, + byte[] value, RowMutations mutation) throws IOException { return checkAndMutate(row, family, qualifier, getCompareOp(op), value, mutation); } @@ -375,7 +376,8 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, } @Override - public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, Put put) throws IOException { + public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, + byte[] value, Put put) throws IOException { return checkAndPut(row, family, qualifier, getCompareOp(op), value, put); } @@ -407,7 +409,8 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, } @Override - public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, Delete delete) throws IOException { + public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, + byte[] value, Delete delete) throws IOException { return ohTable.checkAndDelete(row, family, qualifier, getCompareOp(op), value, delete); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java index b8e1c6b9..5e01b8fe 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java @@ -805,27 +805,9 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, return table.checkAndPut(row, family, qualifier, compareOp, value, put); } - /** - * Atomically checks if a row/family/qualifier value matches the expected - * value. If it does, it adds the put. If the passed value is null, the check - * is for the lack of column (ie: non-existence) - *

- * The expected value argument of this call is on the left and the current - * value of the cell is on the right side of the comparison operator. - *

- * Ie. eg. GREATER operator means expected value > existing <=> add the put. - * - * @param row to check - * @param family column family to check - * @param qualifier column qualifier to check - * @param op comparison operator to use - * @param value the expected value - * @param put data to put if check succeeds - * @return true if the new put was executed, false otherwise - * @throws IOException e - */ @Override - public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, Put put) throws IOException { + public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, + byte[] value, Put put) throws IOException { return checkAndPut(row, family, qualifier, getCompareOp(op), value, put); } @@ -852,27 +834,10 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, return table.checkAndDelete(row, family, qualifier, compareOp, value, delete); } - /** - * Atomically checks if a row/family/qualifier value matches the expected - * value. If it does, it adds the delete. If the passed value is null, the - * check is for the lack of column (ie: non-existence) - *

- * The expected value argument of this call is on the left and the current - * value of the cell is on the right side of the comparison operator. - *

- * Ie. eg. GREATER operator means expected value > existing <=> add the delete. - * - * @param row to check - * @param family column family to check - * @param qualifier column qualifier to check - * @param op comparison operator to use - * @param value the expected value - * @param delete data to delete if check succeeds - * @return true if the new delete was executed, false otherwise - * @throws IOException e - */ @Override - public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, Delete delete) throws IOException { + public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, + CompareOperator op, byte[] value, Delete delete) + throws IOException { return checkAndDelete(row, family, qualifier, getCompareOp(op), value, delete); } @@ -893,11 +858,6 @@ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, lo return table.incrementColumnValue(row, family, qualifier, amount, durability); } - /** - * Returns the actual table back to the pool - * - * @throws IOException if failed - */ public void close() throws IOException { returnTable(table); } @@ -981,27 +941,10 @@ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, return table.checkAndMutate(row, family, qualifier, compareOp, value, mutations); } - /** - * Atomically checks if a row/family/qualifier value matches the expected value. - * If it does, it performs the row mutations. If the passed value is null, the check - * is for the lack of column (ie: non-existence) - *

- * The expected value argument of this call is on the left and the current - * value of the cell is on the right side of the comparison operator. - *

- * Ie. eg. GREATER operator means expected value > existing <=> perform row mutations. - * - * @param row to check - * @param family column family to check - * @param qualifier column qualifier to check - * @param op the comparison operator - * @param value the expected value - * @param mutation mutations to perform if check succeeds - * @return true if the new put was executed, false otherwise - * @throws IOException e - */ @Override - public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op, byte[] value, RowMutations mutation) throws IOException { + public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, + CompareOperator op, byte[] value, RowMutations mutation) + throws IOException { return checkAndMutate(row, family, qualifier, getCompareOp(op), value, mutation); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index deb94f75..a69aa7c5 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -127,7 +127,7 @@ public final class OHConstants { /** * use to specify whether to query only the data in hot storage when performing a query. */ - public static final String HBASE_HTABLE_QUERY_HOT_ONLY = "hbase.htable.query.hot_only"; + public static final String HBASE_HTABLE_QUERY_HOT_ONLY = "hbase.htable.query.hot_only"; /*-------------------------------------------------------------------------------------------------------------*/ diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java index 1c5d51d4..30558872 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java @@ -35,10 +35,7 @@ public T execute(ObTableClient client, ObTableMetaRequest request) throws IOExce ObTable table = client.getRandomTable(); ObTableMetaResponse response; try { - response = (ObTableMetaResponse) client.executeWithRetry( - table, - request, - null /*tableName*/ + response = (ObTableMetaResponse) client.executeWithRetry(table, request, null /*tableName*/ ); } catch (Exception e) { throw new IOException("Failed to execute request", e); diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java index d55ff275..bef2211d 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java @@ -25,25 +25,9 @@ import java.io.IOException; public interface ObTableMetaExecutor { - /** - * 执行元数据请求 - * @param request 元数据请求 - * @return 解析后的元数据对象 - * @throws IOException 如果执行失败或解析失败 - */ T execute(ObTableClient client, ObTableMetaRequest request) throws IOException; - /** - * 解析元数据响应, 用户需要重写 - * @param response 元数据响应 - * @return 解析后的元数据对象 - * @throws IOException 如果解析失败 - */ T parse(ObTableMetaResponse response) throws IOException; - /** - * 获取元信息类型, 用户需要重写 - * @return 元信息类型 - */ ObTableRpcMetaType getMetaType() throws IOException; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index 11859135..d27a3f30 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -1,3 +1,20 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.rpc.ObTableClient; @@ -36,6 +53,7 @@ public class OHAdmin implements Admin { private boolean aborted = false; private final OHConnectionImpl connection; private final Configuration conf; + OHAdmin(OHConnectionImpl connection) { this.connection = connection; this.conf = connection.getConfiguration(); @@ -67,13 +85,14 @@ public Connection getConnection() { public boolean tableExists(TableName tableName) throws IOException { try { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); OHTableExistsExecutor executor = new OHTableExistsExecutor(tableClient); return executor.tableExists(tableName.getNameAsString()); } catch (Exception e) { // try to get the original cause Throwable cause = e.getCause(); - while(cause != null && cause.getCause() != null) { + while (cause != null && cause.getCause() != null) { cause = cause.getCause(); } if (cause instanceof ObTableException) { @@ -119,7 +138,8 @@ public HTableDescriptor[] listTables(Pattern pattern, boolean b) throws IOExcept } @Override - public List listTableDescriptors(Pattern pattern, boolean b) throws IOException { + public List listTableDescriptors(Pattern pattern, boolean b) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -154,15 +174,18 @@ public TableName[] listTableNames(String s, boolean b) throws IOException { } @Override - public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotFoundException, IOException { + public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotFoundException, + IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor( + tableName.getNameAsString(), tableClient); try { return executor.getTableDescriptor(); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else if (e.getCause().getMessage().contains("OB_TABLEGROUP_NOT_EXIST")) { throw new TableNotFoundException(tableName); @@ -175,13 +198,15 @@ public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotF @Override public TableDescriptor getDescriptor(TableName tableName) throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor( + tableName.getNameAsString(), tableClient); try { return executor.getTableDescriptor(); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else if (e.getCause().getMessage().contains("OB_TABLEGROUP_NOT_EXIST")) { throw new TableNotFoundException(tableName); @@ -194,13 +219,14 @@ public TableDescriptor getDescriptor(TableName tableName) throws IOException { @Override public void createTable(TableDescriptor tableDescriptor) throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableDescriptor.getTableName(), connectionConf); OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); try { executor.createTable(tableDescriptor, null); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else { throw e; @@ -209,7 +235,8 @@ public void createTable(TableDescriptor tableDescriptor) throws IOException { } @Override - public void createTable(TableDescriptor tableDescriptor, byte[] bytes, byte[] bytes1, int i) throws IOException { + public void createTable(TableDescriptor tableDescriptor, byte[] bytes, byte[] bytes1, int i) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -219,20 +246,22 @@ public void createTable(TableDescriptor tableDescriptor, byte[][] bytes) throws } @Override - public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] bytes) throws IOException { + public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] bytes) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteTable(TableName tableName) throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); try { executor.deleteTable(tableName.getNameAsString()); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else { throw e; @@ -268,13 +297,15 @@ public Future truncateTableAsync(TableName tableName, boolean b) throws IO @Override public void enableTable(TableName tableName) throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); + OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, + ObTableRpcMetaType.HTABLE_ENABLE_TABLE); try { executor.enableTable(tableName.getNameAsString()); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else { throw e; @@ -305,13 +336,15 @@ public Future disableTableAsync(TableName tableName) throws IOException { @Override public void disableTable(TableName tableName) throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_DISABLE_TABLE); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); + OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, + ObTableRpcMetaType.HTABLE_DISABLE_TABLE); try { executor.disableTable(tableName.getNameAsString()); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else { throw e; @@ -341,8 +374,10 @@ public boolean isTableDisabled(TableName tableName) throws IOException { private boolean isDisabled(TableName tableName) throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHTableDescriptorExecutor tableDescriptor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); + OHTableDescriptorExecutor tableDescriptor = new OHTableDescriptorExecutor( + tableName.getNameAsString(), tableClient); return tableDescriptor.isDisable(); } @@ -367,12 +402,15 @@ public Pair getAlterStatus(byte[] bytes) throws IOException { } @Override - public void addColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + public void addColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public Future addColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + public Future addColumnFamilyAsync(TableName tableName, + ColumnFamilyDescriptor columnFamilyDescriptor) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -387,17 +425,22 @@ public void deleteColumnFamily(TableName tableName, byte[] bytes) throws IOExcep } @Override - public Future deleteColumnFamilyAsync(TableName tableName, byte[] bytes) throws IOException { + public Future deleteColumnFamilyAsync(TableName tableName, byte[] bytes) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void modifyColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + public void modifyColumnFamily(TableName tableName, + ColumnFamilyDescriptor columnFamilyDescriptor) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public Future modifyColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + public Future modifyColumnFamilyAsync(TableName tableName, + ColumnFamilyDescriptor columnFamilyDescriptor) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -462,12 +505,15 @@ public void compactRegion(byte[] bytes, byte[] bytes1) throws IOException { } @Override - public void compact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { + public void compact(TableName tableName, CompactType compactType) throws IOException, + InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void compact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { + public void compact(TableName tableName, byte[] bytes, CompactType compactType) + throws IOException, + InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } @@ -502,17 +548,21 @@ public void majorCompactRegion(byte[] bytes, byte[] bytes1) throws IOException { * @throws InterruptedException */ @Override - public void compactRegionServer(ServerName sn, boolean major) throws IOException, InterruptedException { + public void compactRegionServer(ServerName sn, boolean major) throws IOException, + InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void majorCompact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { + public void majorCompact(TableName tableName, CompactType compactType) throws IOException, + InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void majorCompact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { + public void majorCompact(TableName tableName, byte[] bytes, CompactType compactType) + throws IOException, + InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } @@ -607,7 +657,8 @@ public void mergeRegions(byte[] bytes, byte[] bytes1, boolean b) throws IOExcept } @Override - public Future mergeRegionsAsync(byte[] bytes, byte[] bytes1, boolean b) throws IOException { + public Future mergeRegionsAsync(byte[] bytes, byte[] bytes1, boolean b) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -642,7 +693,8 @@ public Future splitRegionAsync(byte[] bytes, byte[] bytes1) throws IOExcep } @Override - public void modifyTable(TableName tableName, TableDescriptor tableDescriptor) throws IOException { + public void modifyTable(TableName tableName, TableDescriptor tableDescriptor) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -652,7 +704,8 @@ public void modifyTable(TableDescriptor tableDescriptor) throws IOException { } @Override - public Future modifyTableAsync(TableName tableName, TableDescriptor tableDescriptor) throws IOException { + public Future modifyTableAsync(TableName tableName, TableDescriptor tableDescriptor) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -734,10 +787,13 @@ public Map getRegionLoad(ServerName serverName) throws IOExc * @throws IOException if a remote or network exception occurs */ @Override - public Map getRegionLoad(ServerName serverName, TableName tableName) throws IOException { + public Map getRegionLoad(ServerName serverName, TableName tableName) + throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHRegionLoadExecutor executor = new OHRegionLoadExecutor(tableName.getNameAsString(), tableClient); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); + OHRegionLoadExecutor executor = new OHRegionLoadExecutor(tableName.getNameAsString(), + tableClient); return executor.getRegionLoad(); } @@ -752,7 +808,8 @@ public void createNamespace(NamespaceDescriptor namespaceDescriptor) throws IOEx } @Override - public Future createNamespaceAsync(NamespaceDescriptor namespaceDescriptor) throws IOException { + public Future createNamespaceAsync(NamespaceDescriptor namespaceDescriptor) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -762,7 +819,8 @@ public void modifyNamespace(NamespaceDescriptor namespaceDescriptor) throws IOEx } @Override - public Future modifyNamespaceAsync(NamespaceDescriptor namespaceDescriptor) throws IOException { + public Future modifyNamespaceAsync(NamespaceDescriptor namespaceDescriptor) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -777,7 +835,8 @@ public Future deleteNamespaceAsync(String s) throws IOException { } @Override - public NamespaceDescriptor getNamespaceDescriptor(String s) throws NamespaceNotFoundException, IOException { + public NamespaceDescriptor getNamespaceDescriptor(String s) throws NamespaceNotFoundException, + IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -816,7 +875,8 @@ public synchronized void close() throws IOException { } @Override - public HTableDescriptor[] getTableDescriptorsByTableName(List list) throws IOException { + public HTableDescriptor[] getTableDescriptorsByTableName(List list) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -872,7 +932,8 @@ public CompactionState getCompactionState(TableName tableName) throws IOExceptio } @Override - public CompactionState getCompactionState(TableName tableName, CompactType compactType) throws IOException { + public CompactionState getCompactionState(TableName tableName, CompactType compactType) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -892,32 +953,45 @@ public long getLastMajorCompactionTimestampForRegion(byte[] bytes) throws IOExce } @Override - public void snapshot(String s, TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { + public void snapshot(String s, TableName tableName) throws IOException, + SnapshotCreationException, + IllegalArgumentException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void snapshot(byte[] bytes, TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { + public void snapshot(byte[] bytes, TableName tableName) throws IOException, + SnapshotCreationException, + IllegalArgumentException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void snapshot(String s, TableName tableName, SnapshotType snapshotType) throws IOException, SnapshotCreationException, IllegalArgumentException { + public void snapshot(String s, TableName tableName, SnapshotType snapshotType) + throws IOException, + SnapshotCreationException, + IllegalArgumentException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void snapshot(SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException, IllegalArgumentException { + public void snapshot(SnapshotDescription snapshotDescription) throws IOException, + SnapshotCreationException, + IllegalArgumentException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void snapshotAsync(SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException { + public void snapshotAsync(SnapshotDescription snapshotDescription) throws IOException, + SnapshotCreationException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public boolean isSnapshotFinished(SnapshotDescription snapshotDescription) throws IOException, HBaseSnapshotException, UnknownSnapshotException { + public boolean isSnapshotFinished(SnapshotDescription snapshotDescription) + throws IOException, + HBaseSnapshotException, + UnknownSnapshotException { throw new FeatureNotSupportedException("does not support yet"); } @@ -937,7 +1011,8 @@ public Future restoreSnapshotAsync(String s) throws IOException, RestoreSn } @Override - public void restoreSnapshot(byte[] bytes, boolean b) throws IOException, RestoreSnapshotException { + public void restoreSnapshot(byte[] bytes, boolean b) throws IOException, + RestoreSnapshotException { throw new FeatureNotSupportedException("does not support yet"); } @@ -947,27 +1022,35 @@ public void restoreSnapshot(String s, boolean b) throws IOException, RestoreSnap } @Override - public void restoreSnapshot(String s, boolean b, boolean b1) throws IOException, RestoreSnapshotException { + public void restoreSnapshot(String s, boolean b, boolean b1) throws IOException, + RestoreSnapshotException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void cloneSnapshot(byte[] bytes, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { + public void cloneSnapshot(byte[] bytes, TableName tableName) throws IOException, + TableExistsException, + RestoreSnapshotException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void cloneSnapshot(String s, TableName tableName, boolean b) throws IOException, TableExistsException, RestoreSnapshotException { + public void cloneSnapshot(String s, TableName tableName, boolean b) throws IOException, + TableExistsException, + RestoreSnapshotException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public void cloneSnapshot(String s, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { + public void cloneSnapshot(String s, TableName tableName) throws IOException, + TableExistsException, + RestoreSnapshotException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public Future cloneSnapshotAsync(String s, TableName tableName) throws IOException, TableExistsException { + public Future cloneSnapshotAsync(String s, TableName tableName) throws IOException, + TableExistsException { throw new FeatureNotSupportedException("does not support yet"); } @@ -977,12 +1060,14 @@ public void execProcedure(String s, String s1, Map map) throws I } @Override - public byte[] execProcedureWithReturn(String s, String s1, Map map) throws IOException { + public byte[] execProcedureWithReturn(String s, String s1, Map map) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @Override - public boolean isProcedureFinished(String s, String s1, Map map) throws IOException { + public boolean isProcedureFinished(String s, String s1, Map map) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -1007,7 +1092,8 @@ public List listTableSnapshots(String s, String s1) throws } @Override - public List listTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { + public List listTableSnapshots(Pattern pattern, Pattern pattern1) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -1076,25 +1162,12 @@ public List getSecurityCapabilities() throws IOException { throw new FeatureNotSupportedException("does not support yet"); } - /** - * Turn the Split or Merge switches on or off. - * - * @param enabled enabled or not - * @param synchronous If true, it waits until current split() call, if outstanding, to return. - * @param switchTypes switchType list {@link MasterSwitchType} - * @return Previous switch value array - */ @Override - public boolean[] splitOrMergeEnabledSwitch(boolean enabled, boolean synchronous, MasterSwitchType... switchTypes) throws IOException { + public boolean[] splitOrMergeEnabledSwitch(boolean enabled, boolean synchronous, + MasterSwitchType... switchTypes) throws IOException { throw new FeatureNotSupportedException("does not support yet"); } - /** - * Query the current state of the switch. - * - * @param switchType - * @return true if the switch is enabled, false otherwise. - */ @Override public boolean splitOrMergeEnabledSwitch(MasterSwitchType switchType) throws IOException { throw new FeatureNotSupportedException("does not support yet"); @@ -1121,7 +1194,8 @@ public ReplicationPeerConfig getReplicationPeerConfig(String s) throws IOExcepti } @Override - public void updateReplicationPeerConfig(String s, ReplicationPeerConfig replicationPeerConfig) throws IOException { + public void updateReplicationPeerConfig(String s, ReplicationPeerConfig replicationPeerConfig) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -1131,7 +1205,8 @@ public List listReplicationPeers() throws IOExceptio } @Override - public List listReplicationPeers(Pattern pattern) throws IOException { + public List listReplicationPeers(Pattern pattern) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -1146,7 +1221,8 @@ public List listDecommissionedRegionServers() throws IOException { } @Override - public void recommissionRegionServer(ServerName serverName, List list) throws IOException { + public void recommissionRegionServer(ServerName serverName, List list) + throws IOException { throw new FeatureNotSupportedException("does not support yet"); } @@ -1166,7 +1242,8 @@ public void disableTableReplication(TableName tableName) throws IOException { } @Override - public void clearCompactionQueues(ServerName serverName, Set set) throws IOException, InterruptedException { + public void clearCompactionQueues(ServerName serverName, Set set) throws IOException, + InterruptedException { throw new FeatureNotSupportedException("does not support yet"); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index d8fdda02..69c8e2e7 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -249,14 +249,13 @@ private void execute(boolean flushAll) throws IOException { } } - @Override public void close() throws IOException { if (closed) { return; } // reset timeout, timeTick and Timer -// disableWriteBufferPeriodicFlush(); + // disableWriteBufferPeriodicFlush(); try { execute(true); } finally { @@ -303,9 +302,6 @@ public void setOperationTimeout(int operationTimeout) { this.ohTable.setOperationTimeout(operationTimeout); } - /** - * Count the mutations which haven't been processed. - */ @VisibleForTesting public int size() { return undealtMutationCount.get(); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index d5d404a6..0ab83451 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -91,7 +91,7 @@ public OHConnectionConfiguration(Configuration conf) { } this.rpcConnectTimeout = rpcConnectTimeout; this.numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.scannerCaching = conf.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, Integer.MAX_VALUE); this.scannerMaxResultSize = conf.getLong( diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index fd666159..811feca8 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -144,8 +144,10 @@ public RegionLocator getRegionLocator(TableName tableName) throws IOException { // need to use new connection configuration // to avoid change the database in original param url by namespace in tableName OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), obTableClient); + ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClientByTableName( + tableName, connectionConf); + OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), + obTableClient); return executor.getRegionLocator(String.valueOf(tableName)); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoadExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoadExecutor.java index a799e41d..c8d7b0d5 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoadExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLoadExecutor.java @@ -1,23 +1,39 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.core.type.TypeReference; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.hbase.RegionLoad; import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class OHRegionLoadExecutor extends AbstractObTableMetaExecutor> { - private final String tableName; + private final String tableName; private final ObTableClient client; + OHRegionLoadExecutor(String tableName, ObTableClient client) { this.tableName = tableName; this.client = client; @@ -33,15 +49,24 @@ public class OHRegionLoadExecutor extends AbstractObTableMetaExecutor parse(ObTableMetaResponse response) throws IOException { Map regionLoadMap = new LinkedHashMap<>(); - JSONObject metrcisJSONObject = JSON.parseObject(response.getData()); - String tableGroupName = metrcisJSONObject.getString("tableName"); - JSONObject regionList = metrcisJSONObject.getJSONObject("regionList"); - List regions = regionList.getJSONArray("regions").toJavaList(Integer.class); - List memTableSizeList = regionList.getJSONArray("memTableSize").toJavaList(Integer.class); - List ssTableSizeList = regionList.getJSONArray("ssTableSize").toJavaList(Integer.class); - if (regions.isEmpty() || regions.size() != memTableSizeList.size() || memTableSizeList.size() != ssTableSizeList.size()) { - throw new IOException("size length has to be the same"); + // use jackson to parse json + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(response.getData())).orElse(null); + if (jsonMap == null) { + throw new IOException("jsonMap is null"); } + JsonNode tableGroupNameNode = Optional.ofNullable(jsonMap.get("tableName")) + .orElseThrow(() -> new IOException("tableName is null")); + String tableGroupName = tableGroupNameNode.asText(); + + JsonNode regionListNode = Optional.ofNullable(jsonMap.get("regionList")) + .orElseThrow(() -> new IOException("regionList is null")); + List regions = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("regions"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("regions is null")); + List memTableSizeList = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("memTableSize"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("memTableSize is null")); + List ssTableSizeList = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("ssTableSize"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("ssTableSize is null")); for (int i = 0; i < regions.size(); ++i) { String name_str = Integer.toString(regions.get(i)); byte[] name = name_str.getBytes(); @@ -60,12 +85,15 @@ public Map parse(ObTableMetaResponse response) throws IOExce public ObTableRpcMetaType getMetaType() throws IOException { return ObTableRpcMetaType.HTABLE_REGION_METRICS; } - + public Map getRegionLoad() throws IOException { - final ObTableMetaRequest request = new ObTableMetaRequest(); + ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); - final Map requestData = new HashMap<>(); + Map requestData = new HashMap<>(); requestData.put("table_name", tableName); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); return execute(client, request); } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java index dcf42681..c0292f45 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -143,10 +143,7 @@ private ObTableMetaResponse innerExecute(ObTableClient client, ObTableMetaReques ObTable table = client.getRandomTable(); ObTableMetaResponse response; try { - response = (ObTableMetaResponse) client.executeWithRetry( - table, - request, - null /*tableName*/ + response = (ObTableMetaResponse) client.executeWithRetry(table, request, null /*tableName*/ ); } catch (Exception e) { throw new IOException("Failed to execute request", e); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index e0d50cea..0c5a8842 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -128,16 +128,19 @@ public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableCli return OB_TABLE_CLIENT_INSTANCE.get(obTableClientKey); } - public static ObTableClient getOrCreateObTableClientByTableName(TableName tableName, OHConnectionConfiguration connectionConfig) throws IllegalArgumentException, - IOException { + public static ObTableClient getOrCreateObTableClientByTableName(TableName tableName, + OHConnectionConfiguration connectionConfig) + throws IllegalArgumentException, + IOException { String tableNameString = tableName.getNameAsString(); - ObTableClient obTableClient = getOrCreateObTableClient( - OHTable.setUserDefinedNamespace(tableNameString, connectionConfig)); + ObTableClient obTableClient = getOrCreateObTableClient(OHTable.setUserDefinedNamespace( + tableNameString, connectionConfig)); ObTableClientManager.initTimeoutAndRetryTimes(obTableClient, connectionConfig); return obTableClient; } - private static void initTimeoutAndRetryTimes(ObTableClient obTableClient, OHConnectionConfiguration ohConnectionConf) { + private static void initTimeoutAndRetryTimes(ObTableClient obTableClient, + OHConnectionConfiguration ohConnectionConf) { obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); obTableClient.setRuntimeRetryTimes(ohConnectionConf.getNumRetries()); obTableClient.setRuntimeMaxWait(ohConnectionConf.getOperationTimeout()); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 734de8a2..dc1593e7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -2630,7 +2630,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( - Bytes.toBytes(family), Bytes.toBytes(column1)); + Bytes.toBytes(family), Bytes.toBytes(column1)); get.setFilter(dependentColumnFilter); r = hTable.get(get); Assert.assertEquals(3, r.rawCells().length); @@ -2657,7 +2657,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(value2))); + new BinaryComparator(toBytes(value2))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(0, r.rawCells().length); @@ -2666,7 +2666,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value1))); + toBytes(value1))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(2, r.rawCells().length); @@ -2678,7 +2678,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(2, r.rawCells().length); @@ -2687,7 +2687,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); @@ -2696,7 +2696,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( - toBytes(value1))); + toBytes(value1))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(2, r.rawCells().length); @@ -2705,7 +2705,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, - new BinaryComparator(toBytes(value1))); + new BinaryComparator(toBytes(value1))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); @@ -2714,7 +2714,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( - toBytes(value3))); + toBytes(value3))); get.setFilter(valueFilter); r = hTable.get(get); Assert.assertEquals(0, r.rawCells().length); @@ -2736,7 +2736,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(column1))); + new BinaryComparator(toBytes(column1))); get.setFilter(qualifierFilter); r = hTable.get(get); Assert.assertEquals(3, r.rawCells().length); @@ -2745,7 +2745,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(column2))); + toBytes(column2))); get.setFilter(qualifierFilter); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); @@ -2754,7 +2754,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER, - new BinaryComparator(toBytes(column1))); + new BinaryComparator(toBytes(column1))); get.setFilter(qualifierFilter); r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); @@ -2763,7 +2763,7 @@ public void testGetFilter() throws Exception { get.setMaxVersions(10); get.addFamily(toBytes(family)); qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, - new BinaryComparator(toBytes(column1))); + new BinaryComparator(toBytes(column1))); get.setFilter(qualifierFilter); r = hTable.get(get); Assert.assertEquals(7, r.rawCells().length); @@ -2807,7 +2807,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2817,7 +2817,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2827,7 +2827,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), false)); + .toBytes(column1), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2837,7 +2837,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false)); + .toBytes(column2), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2847,7 +2847,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2))); + .toBytes(column2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2857,7 +2857,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), true)); + .toBytes(column2), true)); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2867,8 +2867,8 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2)))); + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2)))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2879,7 +2879,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, - new BinaryComparator(toBytes(column2)))); + new BinaryComparator(toBytes(column2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2890,7 +2890,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(MUST_PASS_ONE); filterList.addFilter(new ColumnCountGetFilter(2)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(column2)))); + new BinaryComparator(toBytes(column2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2901,7 +2901,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(2)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(column2)))); + new BinaryComparator(toBytes(column2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2922,7 +2922,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(column2)))); + new BinaryComparator(toBytes(column2)))); filterList.addFilter(new ColumnCountGetFilter(2)); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -2934,7 +2934,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(2)); filterList.addFilter(new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2)))); + toBytes(value2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2993,7 +2993,7 @@ public void testGetFilter() throws Exception { // 任何一个版本满足则返回本行 SingleColumnValueFilter singleColumnValueFilter; singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3004,7 +3004,7 @@ public void testGetFilter() throws Exception { SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3014,7 +3014,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(4, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3024,7 +3024,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(0, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3035,7 +3035,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value1))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3046,7 +3046,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(0, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3057,7 +3057,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3068,7 +3068,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3079,7 +3079,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -3096,7 +3096,7 @@ public void testGetFilter() throws Exception { tryPut(hTable, putKey1Column2Value2); valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); SkipFilter skipFilter = new SkipFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3106,7 +3106,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(2, r.rawCells().length); valueFilter = new ValueFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); skipFilter = new SkipFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3119,7 +3119,7 @@ public void testGetFilter() throws Exception { tryPut(hTable, putKey1Column2Value1); valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); skipFilter = new SkipFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3129,7 +3129,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(0, r.rawCells().length); valueFilter = new ValueFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); skipFilter = new SkipFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3154,7 +3154,7 @@ public void testGetFilter() throws Exception { WhileMatchFilter whileMatchFilter; valueFilter = new ValueFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator( - toBytes(value2))); + toBytes(value2))); whileMatchFilter = new WhileMatchFilter(valueFilter); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -4988,7 +4988,8 @@ public void testCheckAndDelete() throws IOException { Assert.assertEquals(2, r.rawCells().length); delete = new Delete(key.getBytes()); delete.addColumns(family.getBytes(), column.getBytes()); - ret = hTable.checkAndDelete(key.getBytes(), family.getBytes(), column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), delete); + ret = hTable.checkAndDelete(key.getBytes(), family.getBytes(), column.getBytes(), + CompareFilter.CompareOp.EQUAL, value.getBytes(), delete); Assert.assertTrue(ret); get = new Get(key.getBytes()); get.setMaxVersions(Integer.MAX_VALUE); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index d18d85de..f336d887 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -81,7 +81,8 @@ public void testRefreshTableEntry() throws Exception { @After public void after() throws IOException { - if (hTable != null) hTable.close(); + if (hTable != null) + hTable.close(); } @Test @@ -841,7 +842,6 @@ public void testBufferedMutatorConcurrent() throws Exception { } } - /* CREATE TABLEGROUP test_region_locator SHARDING = 'ADAPTIVE'; CREATE TABLE `test_region_locator$family_region_locator` ( @@ -989,7 +989,7 @@ public void testRangePartitionWithRegionLocator() throws Exception { }); } } - + @Test public void testHRegionLocation() throws IOException { final String tableNameStr = "test_region_locator"; @@ -1034,7 +1034,7 @@ public void testHRegionLocation() throws IOException { }); } } - + @Test public void testKeyPartitionWithRegionLocator() throws IOException { final String tableNameStr = "test_multi_cf"; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 7f1de206..88406e3c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -32,7 +32,6 @@ import org.junit.Assert; import org.junit.Test; - import java.io.IOException; import java.sql.SQLException; import java.sql.Statement; @@ -46,7 +45,7 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; import static org.junit.Assert.assertFalse; -import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; public class OHTableAdminInterfaceTest { public OHTablePool setUpLoadPool() throws IOException { @@ -69,17 +68,16 @@ public OHTablePool setUpPool() throws IOException { } enum ErrSimPoint { - EN_CREATE_HTABLE_TG_FINISH_ERR(2621), - EN_CREATE_HTABLE_CF_FINISH_ERR(2622), - EN_DISABLE_HTABLE_CF_FINISH_ERR(2623), - EN_DELETE_HTABLE_CF_FINISH_ERR(2624); - + EN_CREATE_HTABLE_TG_FINISH_ERR(2621), EN_CREATE_HTABLE_CF_FINISH_ERR(2622), EN_DISABLE_HTABLE_CF_FINISH_ERR( + 2623), EN_DELETE_HTABLE_CF_FINISH_ERR( + 2624); + private final int errCode; - + ErrSimPoint(int errCode) { this.errCode = errCode; } - + public int getErrCode() { return errCode; } @@ -88,18 +86,16 @@ public int getErrCode() { private void setErrSimPoint(ErrSimPoint errSimPoint, boolean enable) { java.sql.Connection connection = null; java.sql.Statement statement = null; - + try { connection = ObHTableTestUtil.getSysTenantConnection(); statement = connection.createStatement(); - + String sql = String.format( "alter system set_tp tp_no = %d, error_code = 4016, frequency = %d", - errSimPoint.getErrCode(), - enable ? 1 : 0 - ); - - statement.execute(sql); + errSimPoint.getErrCode(), enable ? 1 : 0); + + statement.execute(sql); } catch (Exception e) { throw new RuntimeException("Error injection setup failed", e); } finally { @@ -322,7 +318,8 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } - public static void createTable(Admin admin, TableName tableName, String... columnFamilies) throws IOException { + public static void createTable(Admin admin, TableName tableName, String... columnFamilies) + throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); // Add column families for (String cf : columnFamilies) { @@ -772,7 +769,8 @@ public void testCreateDeleteTable() throws Exception { assertTrue(admin.tableExists(tableName)); // TODO: show create table, need to be replace by getDescriptor java.sql.Connection conn = ObHTableTestUtil.getConnection(); - String selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf1); + String selectSql = "show create table " + tableName.getNameAsString() + "$" + + Bytes.toString(cf1); System.out.println("execute sql: " + selectSql); java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); ResultSetPrinter.print(resultSet); @@ -787,7 +785,6 @@ public void testCreateDeleteTable() throws Exception { resultSet = conn.createStatement().executeQuery(selectSql); ResultSetPrinter.print(resultSet); - // 4. test put/get some data Table table = connection.getTable(tableName); Put put = new Put(toBytes("Key" + 1)); @@ -1087,11 +1084,14 @@ public void testHTableDDLDefense() throws Exception { /// execute the following ddl stmt in created by admin table, should be prohibited // 3. alter table add constraint try { - executeSQL(conn, "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)", true); + executeSQL(conn, + "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 4. alter table add index @@ -1100,7 +1100,9 @@ public void testHTableDDLDefense() throws Exception { fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 5. alter table modify column to lob @@ -1109,42 +1111,64 @@ public void testHTableDDLDefense() throws Exception { fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 6. alter hbase admin table add fk try { - executeSQL(conn, "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)", true); + executeSQL( + conn, + "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)", + true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 7. create a normal table to refer to hbase admin table try { - executeSQL(conn, "create table testHTableDefense_t1(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K));" , true); + executeSQL( + conn, + "create table testHTableDefense_t1(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K));", + true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 8. alter a normal table to refer to hbase admin table try { - executeSQL(conn, "create table testHTableDefense_t2(a varbinary(1024) primary key)", true); - executeSQL(conn, "alter table testHTableDefense_t2 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K);", true); + executeSQL(conn, + "create table testHTableDefense_t2(a varbinary(1024) primary key)", true); + executeSQL( + conn, + "alter table testHTableDefense_t2 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K);", + true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 9. create a normal table A to refer to a table mock parent table B, and create table B using hbase admin try { executeSQL(conn, "SET foreign_key_checks = 0", true); - executeSQL(conn, "create table testHTableDefense_t3(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense2$cf1(K));", true); - HTableDescriptor htd2 = new HTableDescriptor(TableName.valueOf("testHTableDefense2")); + executeSQL( + conn, + "create table testHTableDefense_t3(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense2$cf1(K));", + true); + HTableDescriptor htd2 = new HTableDescriptor( + TableName.valueOf("testHTableDefense2")); HColumnDescriptor hcd4 = new HColumnDescriptor("cf1".getBytes()); hcd4.setMaxVersions(2); hcd4.setTimeToLive(172800); @@ -1152,66 +1176,84 @@ public void testHTableDDLDefense() throws Exception { admin.createTable(htd2); fail(); } catch (Exception e) { - Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); } - // 10. create trigger try { - executeSQL(conn, " CREATE TRIGGER hbase_trigger_1" + - " AFTER INSERT ON testHTableDefense$cf1 FOR EACH ROW" + - " BEGIN END", true); + executeSQL(conn, " CREATE TRIGGER hbase_trigger_1" + + " AFTER INSERT ON testHTableDefense$cf1 FOR EACH ROW" + + " BEGIN END", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 11. create view try { - executeSQL(conn, " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); + executeSQL(conn, + " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 12. alter view try { - executeSQL(conn, "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); + executeSQL(conn, "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1", + true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 13. create index try { - executeSQL(conn, " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)", true); + executeSQL(conn, + " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } - // 14. explicit create table and specify created_by:admin, should be prohibited try { - executeSQL(conn, "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", true); + executeSQL( + conn, + "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", + true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + Assert.assertEquals( + "table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", + e.getMessage()); } // 15. alter table to created_by:admin, should be prohibited try { executeSQL(conn, "CREATE TABLE testHTableDefense$cf3(a int primary key)", true); - executeSQL(conn, "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", true); + executeSQL( + conn, + "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", + true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("alter table kv attributes to created by admin not supported", e.getMessage()); + Assert.assertEquals("alter table kv attributes to created by admin not supported", + e.getMessage()); // clean table String sql3 = "drop table if exists testHTableDefense$cf3"; System.out.println("execute sql: " + sql3); @@ -1221,27 +1263,32 @@ public void testHTableDDLDefense() throws Exception { // 16. disable a htable did not created by admin is not suppported try { executeSQL(conn, "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2", true); - executeSQL(conn, "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", true); + executeSQL( + conn, + "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", + true); admin.disableTable(TableName.valueOf("testHTableDefense2")); fail(); } catch (Exception e) { - Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); } // 17. delete a htable did not created by admin is not suppported try { executeSQL(conn, "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2", true); - executeSQL(conn, - "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", true); + executeSQL( + conn, + "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", + true); admin.deleteTable(TableName.valueOf("testHTableDefense2")); fail(); } catch (Exception e) { - Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); } } catch (Exception e) { - e.printStackTrace(); - assertTrue(false); + e.printStackTrace(); + assertTrue(false); } finally { admin.disableTable(tableName); admin.deleteTable(tableName); @@ -1256,13 +1303,15 @@ public void testHTableDDLDefense() throws Exception { void checkKVAttributes(String tableName, String kvAttributes) throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); - java.sql.ResultSet resultSet = conn.createStatement().executeQuery("select kv_attributes from oceanbase.__all_table where table_name = '" + tableName + "'"); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery( + "select kv_attributes from oceanbase.__all_table where table_name = '" + tableName + + "'"); resultSet.next(); String value = resultSet.getString(1); Assert.assertEquals(kvAttributes, value); Assert.assertFalse(resultSet.next()); } - + // NOTE: observer should build with `-DOB_ERRSIM=ON` option, otherwise the test will fail // This test verifies error injection scenarios for table operations @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java index ce1bc3a1..f69143bb 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java @@ -70,7 +70,6 @@ public void testNew() throws Exception { assertTrue(true); } - /* CREATE TABLEGROUP test_desc SHARDING = 'ADAPTIVE'; CREATE TABLE `test_desc$family1` ( diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java index 8e1804e9..07606bae 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -571,7 +571,7 @@ public void testMultiCFDeleteFamily() throws Throwable { public void testMultiCFDeleteFamilyWithVersion() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyWithVersionImpl); } - + @Test public void testMultiCFDeleteFamilyVersion() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyVersionImpl); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java index 2bd69f1f..0ff6a31b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java @@ -1,3 +1,20 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.secondary; import com.alipay.oceanbase.hbase.OHTableClient; diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 2ee37e4e..c1e4ede7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -59,7 +59,7 @@ public static void createTables(TableTemplateManager.TableType type, List tableNames, Map> group2tableNames, boolean printSql) - throws Exception { + throws Exception { Connection conn = ObHTableTestUtil.getConnection(); // single cf table if (tableNames != null) { @@ -110,15 +110,15 @@ public static void alterTables(Connection conn, TableTemplateManager.TableType t System.out.println(sql); conn.createStatement().execute(sql); System.out.println("============= alter table: " + tableName + " table_group: " - + getTableName(tableName) + " =============\n" - + (printSql ? sql : "") - + " \n============= done =============\n"); + + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + + " \n============= done =============\n"); } catch (SQLSyntaxErrorException e) { if (!e.getMessage().contains("already exists")) { throw e; } else { System.out.println("============= table: " + tableName + " table_group: " - + getTableName(tableName) + " alter failed ============="); + + getTableName(tableName) + " alter failed ============="); } } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index ef6cf8e3..f3bdc05a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -35,35 +35,39 @@ public class ObHTableTestUtil { // please consult your dba for the following configuration. - public static String PARAM_URL = ""; - public static String FULL_USER_NAME = ""; - public static String PASSWORD = ""; - public static String SYS_USER_NAME = ""; - public static String SYS_PASSWORD = ""; - public static String ODP_ADDR = ""; - public static int ODP_PORT = 0; - public static boolean ODP_MODE = false; - public static String DATABASE = ""; - public static String JDBC_IP = ""; - public static String JDBC_PORT = ""; - public static String JDBC_DATABASE = ""; - public static String JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " - + JDBC_DATABASE + "?" + "useUnicode=TRUE&" - + "characterEncoding=utf-8&" - + "socketTimeout=3000000&" + "connectTimeout=60000"; - public static String SYS_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " - + "oceanbase?" + "useUnicode=TRUE&" - + "characterEncoding=utf-8&" - + "socketTimeout=3000000&" + "connectTimeout=60000"; - public static String SYS_TENANT_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " - + "oceanbase?" + "useUnicode=TRUE&" - + "characterEncoding=utf-8&" - + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String PARAM_URL = ""; + public static String FULL_USER_NAME = ""; + public static String PASSWORD = ""; + public static String SYS_USER_NAME = ""; + public static String SYS_PASSWORD = ""; + public static String ODP_ADDR = ""; + public static int ODP_PORT = 0; + public static boolean ODP_MODE = false; + public static String DATABASE = ""; + public static String JDBC_IP = ""; + public static String JDBC_PORT = ""; + public static String JDBC_DATABASE = ""; + public static String JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + + "/ " + JDBC_DATABASE + "?" + + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + + "connectTimeout=60000"; + public static String SYS_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + + "/ " + "oceanbase?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + + "connectTimeout=60000"; + public static String SYS_TENANT_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + + "/ " + "oceanbase?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + + "connectTimeout=60000"; public static String SYS_TENANT_USER_NAME = "root@sys"; - public static String SYS_TENANT_PASSWORD = ""; - public static String SQL_FORMAT = "truncate %s"; - public static List tableNameList = new LinkedList(); + public static String SYS_TENANT_PASSWORD = ""; + public static String SQL_FORMAT = "truncate %s"; + public static List tableNameList = new LinkedList(); public static Connection conn; public static Statement stmt; @@ -181,8 +185,8 @@ static public Connection getSysConnection() { static public Connection getSysTenantConnection() { try { Class.forName("com.mysql.cj.jdbc.Driver"); - Connection conn = DriverManager - .getConnection(SYS_TENANT_JDBC_URL, SYS_TENANT_USER_NAME, SYS_TENANT_PASSWORD); + Connection conn = DriverManager.getConnection(SYS_TENANT_JDBC_URL, + SYS_TENANT_USER_NAME, SYS_TENANT_PASSWORD); return conn; } catch (Exception e) { throw new RuntimeException(e); @@ -247,7 +251,8 @@ public static boolean secureCompare(byte[] a, byte[] b) { return diff == 0; } - public static void executeSQL(Connection conn, String sql, boolean printSQL) throws SQLException { + public static void executeSQL(Connection conn, String sql, boolean printSQL) + throws SQLException { System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); } @@ -261,16 +266,16 @@ public static void executeIgnoreUnexpectedError(CheckedRunnable operation) throw executeIgnoreExpectedErrors(operation, "OB_ERR_UNEXPECTED"); } - public static void executeIgnoreExpectedErrors(CheckedRunnable operation, String... expectedErrorMessages) throws Exception { + public static void executeIgnoreExpectedErrors(CheckedRunnable operation, + String... expectedErrorMessages) + throws Exception { try { operation.run(); } catch (Exception e) { boolean shouldIgnore = false; - String[] messagesToCheck = { - e.getMessage(), - e.getCause() != null ? e.getCause().getMessage() : null - }; - + String[] messagesToCheck = { e.getMessage(), + e.getCause() != null ? e.getCause().getMessage() : null }; + for (String expectedMessage : expectedErrorMessages) { for (String actualMessage : messagesToCheck) { if (actualMessage != null && actualMessage.contains(expectedMessage)) { @@ -282,7 +287,7 @@ public static void executeIgnoreExpectedErrors(CheckedRunnable operation, String break; } } - + if (!shouldIgnore) { throw e; } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 2263bfcc..f315471d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -46,66 +46,66 @@ public enum TableType { NON_PARTITIONED_TIME_CELL_TTL, // 时序表带CELL TTL列 } - public static List NORMAL_AND_SERIES_TABLES = Arrays - .asList( - NON_PARTITIONED_REGULAR, - NON_PARTITIONED_TIME_SERIES, - SINGLE_PARTITIONED_REGULAR, - SINGLE_PARTITIONED_TIME_SERIES, - SECONDARY_PARTITIONED_RANGE_KEY, - SECONDARY_PARTITIONED_RANGE_KEY_GEN, - SECONDARY_PARTITIONED_KEY_RANGE, - SECONDARY_PARTITIONED_KEY_RANGE_GEN, - SECONDARY_PARTITIONED_TIME_RANGE_KEY, - SECONDARY_PARTITIONED_TIME_KEY_RANGE); - - public static List NORMAL_PARTITIONED_TABLES = Arrays - .asList( - SINGLE_PARTITIONED_REGULAR, - SECONDARY_PARTITIONED_RANGE_KEY, - SECONDARY_PARTITIONED_RANGE_KEY_GEN, - SECONDARY_PARTITIONED_KEY_RANGE, - SECONDARY_PARTITIONED_KEY_RANGE_GEN); + public static List NORMAL_AND_SERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR, + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + public static List NORMAL_PARTITIONED_TABLES = Arrays + .asList( + SINGLE_PARTITIONED_REGULAR, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN); public static List NORMAL_SERIES_PARTITIONED_TABLES = Arrays - .asList( - SINGLE_PARTITIONED_TIME_SERIES, - SECONDARY_PARTITIONED_TIME_RANGE_KEY, - SECONDARY_PARTITIONED_TIME_KEY_RANGE); - - public static List SERIES_TABLES = Arrays - .asList( - NON_PARTITIONED_TIME_SERIES, - SINGLE_PARTITIONED_TIME_SERIES, - SECONDARY_PARTITIONED_TIME_RANGE_KEY, - SECONDARY_PARTITIONED_TIME_KEY_RANGE); - - public static List NORMAL_TABLES = Arrays - .asList( - NON_PARTITIONED_REGULAR, - SINGLE_PARTITIONED_REGULAR, - SECONDARY_PARTITIONED_RANGE_KEY, - SECONDARY_PARTITIONED_RANGE_KEY_GEN, - SECONDARY_PARTITIONED_KEY_RANGE, - SECONDARY_PARTITIONED_KEY_RANGE_GEN); - - public static List CELL_TTL_TABLES = Arrays - .asList( - NON_PARTITIONED_REGULAR_CELL_TTL, - SINGLE_PARTITIONED_REGULAR_CELL_TTL, - SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, - SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, - SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, - SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL); - - public static List TIMESERIES_TABLES = Arrays - .asList( - NON_PARTITIONED_TIME_SERIES, - SINGLE_PARTITIONED_TIME_SERIES, - SECONDARY_PARTITIONED_TIME_RANGE_KEY, - SECONDARY_PARTITIONED_TIME_KEY_RANGE); - - private static final Map SQL_TEMPLATES = new EnumMap( - TableType.class); + .asList( + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + public static List SERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + public static List NORMAL_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_REGULAR, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN); + + public static List CELL_TTL_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR_CELL_TTL, + SINGLE_PARTITIONED_REGULAR_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL); + + public static List TIMESERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + private static final Map SQL_TEMPLATES = new EnumMap( + TableType.class); static { // 普通表非分区表模版 @@ -297,55 +297,55 @@ public enum TableType { + ") TABLEGROUP = %s"); } - private static final Map ALTER_SQL_TEMPLATES = new EnumMap( - TableType.class); + private static final Map ALTER_SQL_TEMPLATES = new EnumMap( + TableType.class); static { // 普通表一级分区模板 ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, - "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); // 时序表一级分区模板 ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, - "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); // 普通表RANGE-KEY分区（使用K） ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); // 合并GEN类型的注释处理 ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); // 普通表KEY-RANGE分区（使用K） ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); // 普通表KEY-RANGE分区（使用生成列） ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); // 时序表RANGE-KEY分区 ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); // 时序表KEY-RANGE分区 ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); /* ------------------ CELL TTL ----------------*/ ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR_CELL_TTL, - "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, - "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); } public static String getCreateTableSQL(TableType type, String tableName,