Skip to content

Commit d5108b3

Browse files
author
chengyitian
committed
AJ-809: optimize performance for array vector deserialize;
1 parent 424ab40 commit d5108b3

22 files changed

+228
-24
lines changed

‎src/com/xxdb/data/AbstractVector.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
public abstract class AbstractVector extends AbstractEntity implements Vector{
1313
private DATA_FORM df_;
1414
protected int compressedMethod = Vector.COMPRESS_LZ4;
15+
protected static final double GROWTH_FACTOR = 1.5;
1516

1617
public static class NumElementAndPartial{
1718
public int numElement;
@@ -107,6 +108,7 @@ public void write(ExtendedDataOutput out) throws IOException{
107108
public abstract int getUnitLength();
108109
public abstract void Append(Scalar value) throws Exception;
109110
public abstract void Append(Vector value) throws Exception;
111+
public abstract void checkCapacity(int requiredCapacity);
110112

111113
public static boolean checkCompressedMethod(DATA_TYPE type, int compressedMethod) {
112114
if(compressedMethod==COMPRESS_DELTA) {

‎src/com/xxdb/data/BasicAnyVector.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,11 @@ public void Append(Vector value) {
122122
throw new RuntimeException("AnyVector not support append");
123123
}
124124

125+
@Override
126+
public void checkCapacity(int requiredCapacity) {
127+
throw new RuntimeException("BasicAnyVector not support checkCapacity.");
128+
}
129+
125130
public String getString(){
126131
StringBuilder sb = new StringBuilder("(");
127132
int size = Math.min(10, rows());

‎src/com/xxdb/data/BasicArrayVector.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,11 @@ public void Append(Vector value) throws Exception{
381381
throw new RuntimeException("Append to arrayctor must be a vector. ");
382382
}
383383

384+
@Override
385+
public void checkCapacity(int requiredCapacity) {
386+
throw new RuntimeException("BasicArrayVector not support checkCapacity.");
387+
}
388+
384389
@Override
385390
public String getJsonString(int rowIndex) {
386391
StringBuilder sb = new StringBuilder("[");

‎src/com/xxdb/data/BasicBooleanVector.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,10 +198,10 @@ public void add(byte value) {
198198
}
199199

200200
public void addRange(byte[] valueList) {
201-
values = Arrays.copyOf(values, valueList.length + values.length);
201+
int requiredCapacity = size + valueList.length;
202+
checkCapacity(requiredCapacity);
202203
System.arraycopy(valueList, 0, values, size, valueList.length);
203204
size += valueList.length;
204-
capaticy = values.length;
205205
}
206206

207207
@Override
@@ -214,6 +214,18 @@ public void Append(Vector value) throws Exception{
214214
addRange(((BasicBooleanVector)value).getdataArray());
215215
}
216216

217+
@Override
218+
public void checkCapacity(int requiredCapacity) {
219+
if (requiredCapacity > values.length) {
220+
int newCapacity = Math.max(
221+
(int)(values.length * GROWTH_FACTOR),
222+
requiredCapacity
223+
);
224+
values = Arrays.copyOf(values, newCapacity);
225+
capaticy = newCapacity;
226+
}
227+
}
228+
217229
public byte[] getdataArray(){
218230
byte[] data = new byte[size];
219231
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicByteVector.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,10 +195,10 @@ public void add(byte value) {
195195
}
196196

197197
public void addRange(byte[] valueList) {
198-
values = Arrays.copyOf(values, valueList.length + values.length);
198+
int requiredCapacity = size + valueList.length;
199+
checkCapacity(requiredCapacity);
199200
System.arraycopy(valueList, 0, values, size, valueList.length);
200201
size += valueList.length;
201-
capaticy = values.length;
202202
}
203203

204204
@Override
@@ -211,6 +211,18 @@ public void Append(Vector value) throws Exception{
211211
addRange(((BasicByteVector)value).getdataArray());
212212
}
213213

214+
@Override
215+
public void checkCapacity(int requiredCapacity) {
216+
if (requiredCapacity > values.length) {
217+
int newCapacity = Math.max(
218+
(int)(values.length * GROWTH_FACTOR),
219+
requiredCapacity
220+
);
221+
values = Arrays.copyOf(values, newCapacity);
222+
capaticy = newCapacity;
223+
}
224+
}
225+
214226
public byte[] getdataArray(){
215227
byte[] data = new byte[size];
216228
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicComplexVector.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,10 +139,10 @@ public void add(Double2 value) {
139139
}
140140

141141
public void addRange(Double2[] valueList) {
142-
values = Arrays.copyOf(values, valueList.length + values.length);
142+
int requiredCapacity = size + valueList.length;
143+
checkCapacity(requiredCapacity);
143144
System.arraycopy(valueList, 0, values, size, valueList.length);
144145
size += valueList.length;
145-
capaticy = values.length;
146146
}
147147

148148
@Override
@@ -155,6 +155,18 @@ public void Append(Vector value) {
155155
addRange(((BasicComplexVector)value).getdataArray());
156156
}
157157

158+
@Override
159+
public void checkCapacity(int requiredCapacity) {
160+
if (requiredCapacity > values.length) {
161+
int newCapacity = Math.max(
162+
(int)(values.length * GROWTH_FACTOR),
163+
requiredCapacity
164+
);
165+
values = Arrays.copyOf(values, newCapacity);
166+
capaticy = newCapacity;
167+
}
168+
}
169+
158170
public Double2[] getdataArray(){
159171
Double2[] data = new Double2[size];
160172
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicDecimal128Vector.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,11 @@ public void Append(Vector value) throws Exception {
439439
}
440440
}
441441

442+
@Override
443+
public void checkCapacity(int requiredCapacity) {
444+
throw new RuntimeException("BasicDecimal128Vector not support checkCapacity.");
445+
}
446+
442447
@JsonIgnore
443448
public BigInteger[] getdataArray() {
444449
BigInteger[] data = new BigInteger[size];

‎src/com/xxdb/data/BasicDecimal32Vector.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,11 @@ public void Append(Vector value) throws Exception{
334334
}
335335
}
336336

337+
@Override
338+
public void checkCapacity(int requiredCapacity) {
339+
throw new RuntimeException("BasicDecimal32Vector not support checkCapacity.");
340+
}
341+
337342
public void setScale(int scale){
338343
this.scale_ = scale;
339344
}

‎src/com/xxdb/data/BasicDecimal64Vector.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,11 @@ public void Append(Vector value) throws Exception{
350350
}
351351
}
352352

353+
@Override
354+
public void checkCapacity(int requiredCapacity) {
355+
throw new RuntimeException("BasicDecimal64Vector not support checkCapacity.");
356+
}
357+
353358
@JsonIgnore
354359
public long[] getdataArray(){
355360
long[] data = new long[size];

‎src/com/xxdb/data/BasicDoubleVector.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -207,10 +207,10 @@ public void add(double value) {
207207

208208

209209
public void addRange(double[] valueList) {
210-
values = Arrays.copyOf(values, valueList.length + values.length);
210+
int requiredCapacity = size + valueList.length;
211+
checkCapacity(requiredCapacity);
211212
System.arraycopy(valueList, 0, values, size, valueList.length);
212213
size += valueList.length;
213-
capaticy = values.length;
214214
}
215215

216216
@Override
@@ -223,6 +223,18 @@ public void Append(Vector value) {
223223
addRange(((BasicDoubleVector)value).getdataArray());
224224
}
225225

226+
@Override
227+
public void checkCapacity(int requiredCapacity) {
228+
if (requiredCapacity > values.length) {
229+
int newCapacity = Math.max(
230+
(int)(values.length * GROWTH_FACTOR),
231+
requiredCapacity
232+
);
233+
values = Arrays.copyOf(values, newCapacity);
234+
capaticy = newCapacity;
235+
}
236+
}
237+
226238
public double[] getdataArray(){
227239
double[] data = new double[size];
228240
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicDurationVector.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,10 +87,10 @@ public void add(int value, int duration) {
8787

8888

8989
public void addRange(int[] valueList) {
90-
values = Arrays.copyOf(values, valueList.length + values.length);
90+
int requiredCapacity = size + valueList.length;
91+
checkCapacity(requiredCapacity);
9192
System.arraycopy(valueList, 0, values, size, valueList.length);
9293
size += valueList.length;
93-
capaticy = values.length;
9494
}
9595

9696
@Override
@@ -103,6 +103,18 @@ public void Append(Vector value) throws Exception{
103103
addRange(((BasicDurationVector)value).getdataArray());
104104
}
105105

106+
@Override
107+
public void checkCapacity(int requiredCapacity) {
108+
if (requiredCapacity > values.length) {
109+
int newCapacity = Math.max(
110+
(int)(values.length * GROWTH_FACTOR),
111+
requiredCapacity
112+
);
113+
values = Arrays.copyOf(values, newCapacity);
114+
capaticy = newCapacity;
115+
}
116+
}
117+
106118
public int[] getdataArray(){
107119
int[] data = new int[size];
108120
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicFloatVector.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,10 +133,10 @@ public void add(float value) {
133133

134134

135135
public void addRange(float[] valueList) {
136-
values = Arrays.copyOf(values, valueList.length + values.length);
136+
int requiredCapacity = size + valueList.length;
137+
checkCapacity(requiredCapacity);
137138
System.arraycopy(valueList, 0, values, size, valueList.length);
138139
size += valueList.length;
139-
capaticy = values.length;
140140
}
141141

142142
@Override
@@ -149,6 +149,18 @@ public void Append(Vector value) throws Exception{
149149
addRange(((BasicFloatVector)value).getdataArray());
150150
}
151151

152+
@Override
153+
public void checkCapacity(int requiredCapacity) {
154+
if (requiredCapacity > values.length) {
155+
int newCapacity = Math.max(
156+
(int)(values.length * GROWTH_FACTOR),
157+
requiredCapacity
158+
);
159+
values = Arrays.copyOf(values, newCapacity);
160+
capaticy = newCapacity;
161+
}
162+
}
163+
152164
public float[] getdataArray(){
153165
float[] data = new float[size];
154166
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicInt128Vector.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,11 @@ public void Append(Vector value) throws Exception{
213213
addRange(((BasicInt128Vector)value).getdataArray());
214214
}
215215

216+
@Override
217+
public void checkCapacity(int requiredCapacity) {
218+
throw new RuntimeException("BasicInt128Vector not support checkCapacity.");
219+
}
220+
216221
public Long2[] getdataArray(){
217222
Long2[] data = new Long2[size];
218223
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicIntVector.java

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,10 +178,10 @@ public void add(int value) {
178178
}
179179

180180
public void addRange(int[] valueList) {
181-
values = Arrays.copyOf(values, valueList.length + values.length);
181+
int requiredCapacity = size + valueList.length;
182+
checkCapacity(requiredCapacity);
182183
System.arraycopy(valueList, 0, values, size, valueList.length);
183184
size += valueList.length;
184-
capaticy = values.length;
185185
}
186186

187187
@Override
@@ -194,6 +194,19 @@ public void Append(Vector value) throws Exception{
194194
addRange(((BasicIntVector)value).getdataArray());
195195
}
196196

197+
@Override
198+
public void checkCapacity(int requiredCapacity) {
199+
if (requiredCapacity > values.length) {
200+
int newCapacity = Math.max(
201+
(int)(values.length * GROWTH_FACTOR),
202+
requiredCapacity
203+
);
204+
values = Arrays.copyOf(values, newCapacity);
205+
capaticy = newCapacity;
206+
}
207+
}
208+
209+
197210
public int[] getdataArray(){
198211
int[] data = new int[size];
199212
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicIotAnyVector.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,11 @@ public void Append(Vector value) {
149149
throw new RuntimeException("IotAnyVector.Append not supported.");
150150
}
151151

152+
@Override
153+
public void checkCapacity(int requiredCapacity) {
154+
throw new RuntimeException("BasicIotAnyVector not support checkCapacity.");
155+
}
156+
152157
public String getString(){
153158
StringBuilder sb = new StringBuilder("[");
154159
for (int i = 0; i < rows(); i++)

‎src/com/xxdb/data/BasicLongVector.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -180,10 +180,10 @@ public void add(long value) {
180180

181181

182182
public void addRange(long[] valueList) {
183-
values = Arrays.copyOf(values, valueList.length + values.length);
183+
int requiredCapacity = size + valueList.length;
184+
checkCapacity(requiredCapacity);
184185
System.arraycopy(valueList, 0, values, size, valueList.length);
185186
size += valueList.length;
186-
capaticy = values.length;
187187
}
188188

189189
@Override
@@ -196,6 +196,18 @@ public void Append(Vector value) throws Exception{
196196
addRange(((BasicLongVector)value).getdataArray());
197197
}
198198

199+
@Override
200+
public void checkCapacity(int requiredCapacity) {
201+
if (requiredCapacity > values.length) {
202+
int newCapacity = Math.max(
203+
(int)(values.length * GROWTH_FACTOR),
204+
requiredCapacity
205+
);
206+
values = Arrays.copyOf(values, newCapacity);
207+
capaticy = newCapacity;
208+
}
209+
}
210+
199211
public long[] getdataArray(){
200212
long[] data = new long[size];
201213
System.arraycopy(values, 0, data, 0, size);

‎src/com/xxdb/data/BasicPointVector.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,10 +177,10 @@ public void add(Double2 value) {
177177

178178

179179
public void addRange(Double2[] valueList) {
180-
values = Arrays.copyOf(values, valueList.length + values.length);
180+
int requiredCapacity = size + valueList.length;
181+
checkCapacity(requiredCapacity);
181182
System.arraycopy(valueList, 0, values, size, valueList.length);
182183
size += valueList.length;
183-
capaticy = values.length;
184184
}
185185

186186
@Override
@@ -193,6 +193,18 @@ public void Append(Vector value) throws Exception{
193193
addRange(((BasicPointVector)value).getdataArray());
194194
}
195195

196+
@Override
197+
public void checkCapacity(int requiredCapacity) {
198+
if (requiredCapacity > values.length) {
199+
int newCapacity = Math.max(
200+
(int)(values.length * GROWTH_FACTOR),
201+
requiredCapacity
202+
);
203+
values = Arrays.copyOf(values, newCapacity);
204+
capaticy = newCapacity;
205+
}
206+
}
207+
196208
public Double2[] getdataArray(){
197209
Double2[] data = new Double2[size];
198210
System.arraycopy(values, 0, data, 0, size);

0 commit comments

Comments
 (0)