Skip to content

Commit dac5faf

Browse files
authored
[Fix] checkAndMutate and get/Scan with filter return -5006 when include special character (#58)
* [Fix] checkAndMutate and get/Scan with filter return -5006 when include special character * [Chore] fix review * [Fix] fix review
1 parent 70113f6 commit dac5faf

File tree

5 files changed

+264
-120
lines changed

5 files changed

+264
-120
lines changed

src/main/java/com/alipay/oceanbase/hbase/OHTable.java

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult;
4040
import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult;
4141
import com.alipay.sofa.common.thread.SofaThreadPoolExecutor;
42-
import com.alipay.oceanbase.hbase.exception.OperationTimeoutException;
4342

4443
import com.google.protobuf.Descriptors;
4544
import com.google.protobuf.Message;
@@ -60,6 +59,7 @@
6059
import org.apache.hadoop.hbase.util.Pair;
6160
import org.slf4j.Logger;
6261

62+
import java.io.ByteArrayOutputStream;
6363
import java.io.IOException;
6464
import java.util.*;
6565
import java.util.concurrent.*;
@@ -74,6 +74,7 @@
7474
import static java.util.concurrent.TimeUnit.SECONDS;
7575
import static org.apache.commons.lang.StringUtils.isBlank;
7676
import static org.apache.commons.lang.StringUtils.isNotBlank;
77+
import static com.alipay.oceanbase.hbase.filter.HBaseFilterUtils.writeBytesWithEscape;
7778

7879
public class OHTable implements HTableInterface {
7980

@@ -721,7 +722,7 @@ private boolean checkAndMutation(byte[] row, byte[] family, byte[] qualifier, by
721722

722723
checkArgument(!mutation.isEmpty(), "mutation is empty");
723724

724-
String filterString = buildCheckAndMutateFilterString(family, qualifier, value);
725+
byte[] filterString = buildCheckAndMutateFilterString(family, qualifier, value);
725726

726727
ObHTableFilter filter = buildObHTableFilter(filterString, null, 1, qualifier);
727728

@@ -1189,11 +1190,11 @@ private String getTestLoadTargetTableName(String tableNameString, String familyS
11891190
}
11901191

11911192
private ObHTableFilter buildObHTableFilter(Filter filter, TimeRange timeRange, int maxVersion,
1192-
Collection<byte[]> columnQualifiers) {
1193+
Collection<byte[]> columnQualifiers) throws IOException {
11931194
ObHTableFilter obHTableFilter = new ObHTableFilter();
11941195

11951196
if (filter != null) {
1196-
obHTableFilter.setFilterString(HBaseFilterUtils.toParseableString(filter));
1197+
obHTableFilter.setFilterString(HBaseFilterUtils.toParseableByteArray(filter));
11971198
}
11981199

11991200
if (timeRange != null) {
@@ -1215,18 +1216,23 @@ private ObHTableFilter buildObHTableFilter(Filter filter, TimeRange timeRange, i
12151216
return obHTableFilter;
12161217
}
12171218

1218-
private String buildCheckAndMutateFilterString(byte[] family, byte[] qualifier, byte[] value) {
1219+
private byte[] buildCheckAndMutateFilterString(byte[] family, byte[] qualifier, byte[] value) throws IOException {
1220+
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
1221+
byteStream.write("CheckAndMutateFilter(=, 'binary:".getBytes());
1222+
writeBytesWithEscape(byteStream, value);
1223+
byteStream.write("', '".getBytes());
1224+
writeBytesWithEscape(byteStream, family);
1225+
byteStream.write("', '".getBytes());
1226+
writeBytesWithEscape(byteStream, qualifier);
12191227
if (value != null) {
1220-
return ("CheckAndMutateFilter(=, 'binary:" + Bytes.toString(value) + "', '"
1221-
+ Bytes.toString(family) + "', '"
1222-
+ (qualifier == null ? "" : Bytes.toString(qualifier)) + "', false)");
1228+
byteStream.write("', false)".getBytes());
12231229
} else {
1224-
return ("CheckAndMutateFilter(=, 'binary:', '" + Bytes.toString(family) + "', '"
1225-
+ (qualifier == null ? "" : Bytes.toString(qualifier)) + "', true)");
1230+
byteStream.write("', true)".getBytes());
12261231
}
1232+
return byteStream.toByteArray();
12271233
}
12281234

1229-
private ObHTableFilter buildObHTableFilter(String filterString, TimeRange timeRange,
1235+
private ObHTableFilter buildObHTableFilter(byte[] filterString, TimeRange timeRange,
12301236
int maxVersion, byte[]... columnQualifiers) {
12311237
ObHTableFilter obHTableFilter = new ObHTableFilter();
12321238

src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java

Lines changed: 126 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -18,152 +18,213 @@
1818
package com.alipay.oceanbase.hbase.filter;
1919

2020
import org.apache.hadoop.hbase.filter.*;
21-
import org.apache.hadoop.hbase.util.Bytes;
2221

22+
import java.io.ByteArrayOutputStream;
23+
import java.io.IOException;
2324
import java.util.List;
2425

2526
public class HBaseFilterUtils {
2627

27-
public static String toParseableString(Filter filter) {
28+
public static byte[] toParseableByteArray(Filter filter) throws IOException {
29+
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
30+
toParseableByteArray(byteStream, filter);
31+
return byteStream.toByteArray();
32+
}
33+
34+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filter filter) throws IOException {
2835
if (filter == null) {
2936
throw new IllegalArgumentException("Filter is null");
3037
} else if (filter instanceof CompareFilter) {
3138
// RowFilter, ValueFilter, QualifierFilter
32-
return toParseableString((CompareFilter) filter);
39+
toParseableByteArray(byteStream, (CompareFilter) filter);
3340
} else if (filter instanceof SingleColumnValueFilter) {
34-
return toParseableString((SingleColumnValueFilter) filter);
41+
toParseableByteArray(byteStream, (SingleColumnValueFilter) filter);
3542
} else if (filter instanceof PageFilter) {
36-
return toParseableString((PageFilter) filter);
43+
toParseableByteArray(byteStream, (PageFilter) filter);
3744
} else if (filter instanceof ColumnCountGetFilter) {
38-
return toParseableString((ColumnCountGetFilter) filter);
45+
toParseableByteArray(byteStream, (ColumnCountGetFilter) filter);
3946
} else if (filter instanceof PrefixFilter) {
40-
return toParseableString((PrefixFilter) filter);
47+
toParseableByteArray(byteStream, (PrefixFilter) filter);
4148
} else if (filter instanceof FilterList) {
42-
return toParseableString((FilterList) filter);
49+
toParseableByteArray(byteStream, (FilterList) filter);
4350
} else if (filter instanceof ColumnPaginationFilter) {
44-
return toParseableString((ColumnPaginationFilter) filter);
51+
toParseableByteArray(byteStream, (ColumnPaginationFilter) filter);
4552
} else if (filter instanceof SkipFilter) {
46-
return toParseableString((SkipFilter) filter);
53+
toParseableByteArray(byteStream, (SkipFilter) filter);
4754
} else if (filter instanceof WhileMatchFilter) {
48-
return toParseableString((WhileMatchFilter) filter);
55+
toParseableByteArray(byteStream, (WhileMatchFilter) filter);
4956
} else {
5057
throw new IllegalArgumentException("Invalid filter: " + filter);
5158
}
5259
}
5360

54-
private static String toParseableString(CompareFilter.CompareOp op) {
61+
private static byte[] toParseableByteArray(CompareFilter.CompareOp op) {
5562
if (op == null) {
5663
throw new IllegalArgumentException("Compare operator is null");
5764
}
5865
switch (op) {
5966
case LESS:
60-
return Bytes.toString(ParseConstants.LESS_THAN_ARRAY);
67+
return ParseConstants.LESS_THAN_ARRAY;
6168
case LESS_OR_EQUAL:
62-
return Bytes.toString(ParseConstants.LESS_THAN_OR_EQUAL_TO_ARRAY);
69+
return ParseConstants.LESS_THAN_OR_EQUAL_TO_ARRAY;
6370
case EQUAL:
64-
return Bytes.toString(ParseConstants.EQUAL_TO_ARRAY);
71+
return ParseConstants.EQUAL_TO_ARRAY;
6572
case NOT_EQUAL:
66-
return Bytes.toString(ParseConstants.NOT_EQUAL_TO_ARRAY);
73+
return ParseConstants.NOT_EQUAL_TO_ARRAY;
6774
case GREATER_OR_EQUAL:
68-
return Bytes.toString(ParseConstants.GREATER_THAN_OR_EQUAL_TO_ARRAY);
75+
return ParseConstants.GREATER_THAN_OR_EQUAL_TO_ARRAY;
6976
case GREATER:
70-
return Bytes.toString(ParseConstants.GREATER_THAN_ARRAY);
77+
return ParseConstants.GREATER_THAN_ARRAY;
7178
default:
7279
throw new IllegalArgumentException("Invalid compare operator: " + op);
7380
}
7481
}
7582

76-
private static String toParseableString(ByteArrayComparable comparator) {
83+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, ByteArrayComparable comparator) throws IOException {
7784
if (comparator == null) {
7885
throw new IllegalArgumentException("Comparator is null");
7986
}
80-
StringBuilder sb = new StringBuilder();
87+
byteStream.write('\'');
8188
if (comparator instanceof BinaryComparator) {
82-
sb.append('\'').append(Bytes.toString(ParseConstants.binaryType)).append(':')
83-
.append(Bytes.toString(comparator.getValue())).append('\'');
89+
byteStream.write(ParseConstants.binaryType);
8490
} else if (comparator instanceof BinaryPrefixComparator) {
85-
sb.append('\'').append(Bytes.toString(ParseConstants.binaryPrefixType)).append(':')
86-
.append(Bytes.toString(comparator.getValue())).append('\'');
91+
byteStream.write(ParseConstants.binaryPrefixType);
8792
} else if (comparator instanceof RegexStringComparator) {
88-
sb.append('\'').append(Bytes.toString(ParseConstants.regexStringType)).append(':')
89-
.append(Bytes.toString(comparator.getValue())).append('\'');
93+
byteStream.write(ParseConstants.regexStringType);
9094
} else if (comparator instanceof SubstringComparator) {
91-
sb.append('\'').append(Bytes.toString(ParseConstants.substringType)).append(':')
92-
.append(Bytes.toString(comparator.getValue())).append('\'');
95+
byteStream.write(ParseConstants.substringType);
9396
} else {
9497
throw new IllegalArgumentException("This comparator has not been implemented "
9598
+ comparator);
9699
}
97-
return sb.toString();
100+
byteStream.write(':');
101+
writeBytesWithEscape(byteStream, comparator.getValue());
102+
byteStream.write('\'');
98103
}
99104

100-
private static String toParseableString(CompareFilter filter) {
101-
return filter.getClass().getSimpleName() + '(' + toParseableString(filter.getOperator())
102-
+ ',' + toParseableString(filter.getComparator()) + ')';
105+
// CompareFilter(=,'binary:123')
106+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, CompareFilter filter) throws IOException {
107+
byteStream.write(filter.getClass().getSimpleName().getBytes());
108+
byteStream.write('(');
109+
byteStream.write(toParseableByteArray(filter.getOperator()));
110+
byteStream.write(',');
111+
toParseableByteArray(byteStream, filter.getComparator());
112+
byteStream.write(')');
103113
}
104114

105-
private static String toParseableString(SingleColumnValueFilter filter) {
106-
return filter.getClass().getSimpleName() + "('" + Bytes.toString(filter.getFamily())
107-
+ "','" + Bytes.toString(filter.getQualifier()) + "',"
108-
+ toParseableString(filter.getOperator()) + ','
109-
+ toParseableString(filter.getComparator()) + ',' + filter.getFilterIfMissing()
110-
+ ',' + filter.getLatestVersionOnly() + ')';
115+
// SingleColumnValueFilter('cf1','col1',=,'binary:123',true,true)
116+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, SingleColumnValueFilter filter) throws IOException {
117+
byteStream.write(filter.getClass().getSimpleName().getBytes());
118+
byteStream.write("('".getBytes());
119+
writeBytesWithEscape(byteStream, filter.getFamily());
120+
byteStream.write("','".getBytes());
121+
writeBytesWithEscape(byteStream, filter.getQualifier());
122+
byteStream.write("',".getBytes());
123+
byteStream.write(toParseableByteArray(filter.getOperator()));
124+
byteStream.write(',');
125+
toParseableByteArray(byteStream, filter.getComparator());
126+
byteStream.write(',');
127+
byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes());
128+
byteStream.write(',');
129+
byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes());
130+
byteStream.write(')');
111131
}
112132

113-
private static String toParseableString(PageFilter filter) {
114-
return filter.getClass().getSimpleName() + '(' + filter.getPageSize() + ')';
133+
// PageFilter(100);
134+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException {
135+
byteStream.write(filter.getClass().getSimpleName().getBytes());
136+
byteStream.write('(');
137+
byteStream.write(Long.toString(filter.getPageSize()).getBytes());
138+
byteStream.write(')');
115139
}
116140

117-
private static String toParseableString(ColumnPaginationFilter filter) {
118-
return filter.getClass().getSimpleName() + '(' + filter.getLimit() + ','
119-
+ filter.getOffset() + ')';
141+
// ColumnPaginationFilter(ColumnPaginationFilter(10,2)
142+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException {
143+
byteStream.write(filter.getClass().getSimpleName().getBytes());
144+
byteStream.write('(');
145+
byteStream.write(Long.toString(filter.getLimit()).getBytes());
146+
byteStream.write(',');
147+
byteStream.write(Long.toString(filter.getOffset()).getBytes());
148+
byteStream .write(')');
120149
}
121150

122-
private static String toParseableString(ColumnCountGetFilter filter) {
123-
return filter.getClass().getSimpleName() + '(' + filter.getLimit() + ')';
151+
// ColumnCountGetFilter(100)
152+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnCountGetFilter filter) throws IOException {
153+
byteStream.write(filter.getClass().getSimpleName().getBytes());
154+
byteStream.write('(');
155+
byteStream.write(Long.toString(filter.getLimit()).getBytes());
156+
byteStream .write(')');
124157
}
125158

126-
private static String toParseableString(PrefixFilter filter) {
127-
return filter.getClass().getSimpleName() + "('" + Bytes.toString(filter.getPrefix()) + "')";
159+
// PrefixFilter('prefix');
160+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, PrefixFilter filter) throws IOException {
161+
byteStream.write(filter.getClass().getSimpleName().getBytes());
162+
byteStream.write("('".getBytes());
163+
writeBytesWithEscape(byteStream, filter.getPrefix());
164+
byteStream .write("')".getBytes());
128165
}
129166

130-
private static String toParseableString(SkipFilter filter) {
131-
return "(" + Bytes.toString(ParseConstants.SKIP_ARRAY) + " "
132-
+ toParseableString(filter.getFilter()) + ")";
167+
// (SKIP filter)
168+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, SkipFilter filter) throws IOException {
169+
byteStream.write('(');
170+
byteStream.write(ParseConstants.SKIP_ARRAY);
171+
byteStream.write(' ');
172+
toParseableByteArray(byteStream, filter.getFilter());
173+
byteStream.write(')');
133174
}
134175

135-
private static String toParseableString(WhileMatchFilter filter) {
136-
return "(" + Bytes.toString(ParseConstants.WHILE_ARRAY) + " "
137-
+ toParseableString(filter.getFilter()) + ")";
176+
// (WHILE filter)
177+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, WhileMatchFilter filter) throws IOException {
178+
byteStream.write('(');
179+
byteStream.write(ParseConstants.WHILE_ARRAY);
180+
byteStream.write(' ');
181+
toParseableByteArray(byteStream, filter.getFilter());
182+
byteStream.write(')');
138183
}
139184

140-
private static String toParseableString(FilterList filterList) {
141-
StringBuilder sb = new StringBuilder();
185+
// (filter and filter ...) or (filter or filter ...)
186+
// when filter list is empty, "" is generated, and empty filter list member is removed
187+
// in result parseable byteArray
188+
private static void toParseableByteArray(ByteArrayOutputStream byteStream, FilterList filterList) throws IOException {
142189
List<Filter> filters = filterList.getFilters();
143190
boolean isEmpty = true;
191+
ByteArrayOutputStream oneFilterBytes = new ByteArrayOutputStream();
144192
for (int i = 0; i < filters.size(); i++) {
145-
String filterString = toParseableString(filters.get(i));
146-
if (filterString.isEmpty())
147-
continue;
193+
toParseableByteArray(oneFilterBytes, filters.get(i));
194+
if (oneFilterBytes.size() == 0) { continue; }
148195
if (isEmpty) {
149-
sb.append("(").append(filterString);
196+
byteStream.write('(');
150197
isEmpty = false;
151198
} else {
152-
sb.append(" ");
199+
byteStream.write(' ');
153200
if (filterList.getOperator().equals(FilterList.Operator.MUST_PASS_ALL)) {
154-
sb.append(Bytes.toString(ParseConstants.AND));
201+
byteStream.write(ParseConstants.AND);
155202
} else if (filterList.getOperator().equals(FilterList.Operator.MUST_PASS_ONE)) {
156-
sb.append(Bytes.toString(ParseConstants.OR));
203+
byteStream.write(ParseConstants.OR);
157204
} else {
158205
throw new IllegalArgumentException("Invalid FilterList: " + filterList);
159206
}
160-
sb.append(" ").append(filterString);
207+
byteStream.write(' ');
161208
}
209+
oneFilterBytes.writeTo(byteStream);
210+
oneFilterBytes.reset();
162211
}
163212
if (!isEmpty) {
164-
sb.append(")");
213+
byteStream.write(')');
165214
}
166-
return sb.toString();
167215
}
168216

217+
// when write family/qualifier/value/row into hbase filter, need add escape for
218+
// special character to prevent parse error in server
219+
public static void writeBytesWithEscape(ByteArrayOutputStream byteStream, byte[] bytes) throws IOException {
220+
if (bytes == null) {
221+
return;
222+
}
223+
for (int i = 0; i < bytes.length; i++) {
224+
if (bytes[i] == '\'') {
225+
byteStream.write('\'');
226+
}
227+
byteStream.write(bytes[i]);
228+
}
229+
}
169230
}

0 commit comments

Comments
 (0)