Skip to content

Commit 22a8381

Browse files
aditanaseadragomir
aditanase
authored and
adragomir
committed
[HSTACK] - fix array_has returning false instead of NULL on empty array
1 parent 254ec03 commit 22a8381

File tree

2 files changed

+57
-20
lines changed

2 files changed

+57
-20
lines changed

‎datafusion/functions-nested/src/array_has.rs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,8 @@ fn array_has_dispatch_for_scalar<O: OffsetSizeTrait>(
220220
let values = haystack.values();
221221
let is_nested = values.data_type().is_nested();
222222
let offsets = haystack.value_offsets();
223+
let nulls = haystack.nulls();
224+
223225
// If first argument is empty list (second argument is non-null), return false
224226
// i.e. array_has([], non-null element) -> false
225227
if values.len() == 0 {
@@ -234,9 +236,15 @@ fn array_has_dispatch_for_scalar<O: OffsetSizeTrait>(
234236
let start = offset[0].to_usize().unwrap();
235237
let end = offset[1].to_usize().unwrap();
236238
let length = end - start;
237-
// For non-nested list, length is 0 for null
239+
// For non-nested list, check null vs empty
240+
// otherwise array_has on [] returns null instead of false
238241
if length == 0 {
239-
continue;
242+
if let Some(nulls) = nulls {
243+
if nulls.is_null(i) {
244+
continue;
245+
}
246+
}
247+
final_contained[i] = Some(false);
240248
}
241249
let sliced_array = eq_array.slice(start, length);
242250
final_contained[i] = Some(sliced_array.true_count() > 0);

‎datafusion/sqllogictest/test_files/array.slt

Lines changed: 47 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ AS VALUES
6363
(make_array(make_array(5, 6),make_array(7, 8)), make_array(7.7, 8.8, 9.9), make_array('d', NULL, 'l', 'o', 'r')),
6464
(make_array(make_array(7, NULL),make_array(9, 10)), make_array(10.1, NULL, 12.2), make_array('s', 'i', 't')),
6565
(NULL, make_array(13.3, 14.4, 15.5), make_array('a', 'm', 'e', 't')),
66+
(NULL, make_array(), make_array('a', 'm', 'e', 't')),
6667
(make_array(make_array(11, 12),make_array(13, 14)), NULL, make_array(',')),
6768
(make_array(make_array(15, 16),make_array(NULL, 18)), make_array(16.6, 17.7, 18.8), NULL)
6869
;
@@ -709,6 +710,7 @@ List(Field { name: "item", data_type: List(Field { name: "item", data_type: Int6
709710
List(Field { name: "item", data_type: List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) List(Field { name: "item", data_type: Float64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) List(Field { name: "item", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} })
710711
List(Field { name: "item", data_type: List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) List(Field { name: "item", data_type: Float64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) List(Field { name: "item", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} })
711712
List(Field { name: "item", data_type: List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) List(Field { name: "item", data_type: Float64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) List(Field { name: "item", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} })
713+
List(Field { name: "item", data_type: List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) List(Field { name: "item", data_type: Float64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) List(Field { name: "item", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} })
712714

713715
# arrays table
714716
query ???
@@ -719,6 +721,7 @@ select column1, column2, column3 from arrays;
719721
[[5, 6], [7, 8]] [7.7, 8.8, 9.9] [d, NULL, l, o, r]
720722
[[7, NULL], [9, 10]] [10.1, NULL, 12.2] [s, i, t]
721723
NULL [13.3, 14.4, 15.5] [a, m, e, t]
724+
NULL [] [a, m, e, t]
722725
[[11, 12], [13, 14]] NULL [,]
723726
[[15, 16], [NULL, 18]] [16.6, 17.7, 18.8] NULL
724727

@@ -844,6 +847,7 @@ select column1[2], column2[3], column3[1] from arrays;
844847
[7, 8] 9.9 d
845848
[9, 10] 12.2 s
846849
NULL 15.5 a
850+
NULL NULL a
847851
[13, 14] NULL ,
848852
[NULL, 18] 18.8 NULL
849853

@@ -858,6 +862,7 @@ NULL NULL NULL
858862
NULL NULL NULL
859863
NULL NULL NULL
860864
NULL NULL NULL
865+
NULL NULL NULL
861866

862867
# single index with columns #3 (negative index)
863868
query ?RT
@@ -868,6 +873,7 @@ select column1[-2], column2[-3], column3[-1] from arrays;
868873
[5, 6] 7.7 r
869874
[7, NULL] 10.1 t
870875
NULL 13.3 t
876+
NULL NULL t
871877
[11, 12] NULL ,
872878
[15, 16] 16.6 NULL
873879

@@ -880,6 +886,7 @@ select column1[9 - 7], column2[2 * 0], column3[1 - 3] from arrays;
880886
[7, 8] NULL o
881887
[9, 10] NULL i
882888
NULL NULL e
889+
NULL NULL e
883890
[13, 14] NULL NULL
884891
[NULL, 18] NULL NULL
885892

@@ -951,6 +958,7 @@ select column1[2:4], column2[1:4], column3[3:4] from arrays;
951958
[[7, 8]] [7.7, 8.8, 9.9] [l, o]
952959
[[9, 10]] [10.1, NULL, 12.2] [t]
953960
NULL [13.3, 14.4, 15.5] [e, t]
961+
NULL [] [e, t]
954962
[[13, 14]] NULL []
955963
[[NULL, 18]] [16.6, 17.7, 18.8] NULL
956964

@@ -963,6 +971,7 @@ select column1[0:5], column2[0:3], column3[0:9] from arrays;
963971
[[5, 6], [7, 8]] [7.7, 8.8, 9.9] [d, NULL, l, o, r]
964972
[[7, NULL], [9, 10]] [10.1, NULL, 12.2] [s, i, t]
965973
NULL [13.3, 14.4, 15.5] [a, m, e, t]
974+
NULL [] [a, m, e, t]
966975
[[11, 12], [13, 14]] NULL [,]
967976
[[15, 16], [NULL, 18]] [16.6, 17.7, 18.8] NULL
968977

@@ -1027,6 +1036,7 @@ select column1[2:4:2], column2[1:4:2], column3[3:4:2] from arrays;
10271036
[[7, 8]] [7.7, 9.9] [l]
10281037
[[9, 10]] [10.1, 12.2] [t]
10291038
NULL [13.3, 15.5] [e]
1039+
NULL [] [e]
10301040
[[13, 14]] NULL []
10311041
[[NULL, 18]] [16.6, 18.8] NULL
10321042

@@ -1039,6 +1049,7 @@ select column1[0:5:2], column2[0:3:2], column3[0:9:2] from arrays;
10391049
[[5, 6]] [7.7, 9.9] [d, l, r]
10401050
[[7, NULL]] [10.1, 12.2] [s, t]
10411051
NULL [13.3, 15.5] [a, e]
1052+
NULL [] [a, e]
10421053
[[11, 12]] NULL [,]
10431054
[[15, 16]] [16.6, 18.8] NULL
10441055

@@ -2488,6 +2499,7 @@ select array_append(column2, 100.1), array_append(column3, '.') from arrays;
24882499
[7.7, 8.8, 9.9, 100.1] [d, NULL, l, o, r, .]
24892500
[10.1, NULL, 12.2, 100.1] [s, i, t, .]
24902501
[13.3, 14.4, 15.5, 100.1] [a, m, e, t, .]
2502+
[100.1] [a, m, e, t, .]
24912503
[100.1] [,, .]
24922504
[16.6, 17.7, 18.8, 100.1] [.]
24932505

@@ -2499,6 +2511,7 @@ select array_append(column2, 100.1), array_append(column3, '.') from large_array
24992511
[7.7, 8.8, 9.9, 100.1] [d, NULL, l, o, r, .]
25002512
[10.1, NULL, 12.2, 100.1] [s, i, t, .]
25012513
[13.3, 14.4, 15.5, 100.1] [a, m, e, t, .]
2514+
[100.1] [a, m, e, t, .]
25022515
[100.1] [,, .]
25032516
[16.6, 17.7, 18.8, 100.1] [.]
25042517

@@ -2743,6 +2756,7 @@ select array_prepend(100.1, column2), array_prepend('.', column3) from arrays;
27432756
[100.1, 7.7, 8.8, 9.9] [., d, NULL, l, o, r]
27442757
[100.1, 10.1, NULL, 12.2] [., s, i, t]
27452758
[100.1, 13.3, 14.4, 15.5] [., a, m, e, t]
2759+
[100.1] [., a, m, e, t]
27462760
[100.1] [., ,]
27472761
[100.1, 16.6, 17.7, 18.8] [.]
27482762

@@ -2754,6 +2768,7 @@ select array_prepend(100.1, column2), array_prepend('.', column3) from large_arr
27542768
[100.1, 7.7, 8.8, 9.9] [., d, NULL, l, o, r]
27552769
[100.1, 10.1, NULL, 12.2] [., s, i, t]
27562770
[100.1, 13.3, 14.4, 15.5] [., a, m, e, t]
2771+
[100.1] [., a, m, e, t]
27572772
[100.1] [., ,]
27582773
[100.1, 16.6, 17.7, 18.8] [.]
27592774

@@ -3112,6 +3127,7 @@ select array_concat(column1, column1), array_concat(column2, column2), array_con
31123127
[[5, 6], [7, 8], [5, 6], [7, 8]] [7.7, 8.8, 9.9, 7.7, 8.8, 9.9] [d, NULL, l, o, r, d, NULL, l, o, r]
31133128
[[7, NULL], [9, 10], [7, NULL], [9, 10]] [10.1, NULL, 12.2, 10.1, NULL, 12.2] [s, i, t, s, i, t]
31143129
NULL [13.3, 14.4, 15.5, 13.3, 14.4, 15.5] [a, m, e, t, a, m, e, t]
3130+
NULL [] [a, m, e, t, a, m, e, t]
31153131
[[11, 12], [13, 14], [11, 12], [13, 14]] NULL [,, ,]
31163132
[[15, 16], [NULL, 18], [15, 16], [NULL, 18]] [16.6, 17.7, 18.8, 16.6, 17.7, 18.8] NULL
31173133

@@ -3124,6 +3140,7 @@ select array_concat(column1, make_array(make_array(1, 2), make_array(3, 4))), ar
31243140
[[5, 6], [7, 8], [1, 2], [3, 4]] [7.7, 8.8, 9.9, 1.1, 2.2, 3.3]
31253141
[[7, NULL], [9, 10], [1, 2], [3, 4]] [10.1, NULL, 12.2, 1.1, 2.2, 3.3]
31263142
[[1, 2], [3, 4]] [13.3, 14.4, 15.5, 1.1, 2.2, 3.3]
3143+
[[1, 2], [3, 4]] [1.1, 2.2, 3.3]
31273144
[[11, 12], [13, 14], [1, 2], [3, 4]] [1.1, 2.2, 3.3]
31283145
[[15, 16], [NULL, 18], [1, 2], [3, 4]] [16.6, 17.7, 18.8, 1.1, 2.2, 3.3]
31293146

@@ -3136,6 +3153,7 @@ select array_concat(column3, make_array('.', '.', '.')) from arrays;
31363153
[d, NULL, l, o, r, ., ., .]
31373154
[s, i, t, ., ., .]
31383155
[a, m, e, t, ., ., .]
3156+
[a, m, e, t, ., ., .]
31393157
[,, ., ., .]
31403158
[., ., .]
31413159

@@ -4544,6 +4562,7 @@ NULL 0
45444562
#NULL 0
45454563

45464564
# cardinality with columns
4565+
# FIXME cardinality on empty array should be NULL or zero?
45474566
query III
45484567
select cardinality(column1), cardinality(column2), cardinality(column3) from arrays;
45494568
----
@@ -4552,6 +4571,7 @@ select cardinality(column1), cardinality(column2), cardinality(column3) from arr
45524571
4 3 5
45534572
4 3 3
45544573
NULL 3 4
4574+
NULL NULL 4
45554575
4 NULL 1
45564576
4 3 NULL
45574577

@@ -4563,6 +4583,7 @@ select cardinality(column1), cardinality(column2), cardinality(column3) from lar
45634583
4 3 5
45644584
4 3 3
45654585
NULL 3 4
4586+
NULL NULL 4
45664587
4 NULL 1
45674588
4 3 NULL
45684589

@@ -5233,6 +5254,7 @@ select array_dims(column1), array_dims(column2), array_dims(column3) from arrays
52335254
[2, 2] [3] [5]
52345255
[2, 2] [3] [3]
52355256
NULL [3] [4]
5257+
NULL NULL [4]
52365258
[2, 2] NULL [1]
52375259
[2, 2] [3] NULL
52385260

@@ -5244,6 +5266,7 @@ select array_dims(column1), array_dims(column2), array_dims(column3) from large_
52445266
[2, 2] [3] [5]
52455267
[2, 2] [3] [3]
52465268
NULL [3] [4]
5269+
NULL NULL [4]
52475270
[2, 2] NULL [1]
52485271
[2, 2] [3] NULL
52495272

@@ -5400,6 +5423,7 @@ select array_ndims(column1), array_ndims(column2), array_ndims(column3) from arr
54005423
2 1 1
54015424
2 1 1
54025425
NULL 1 1
5426+
NULL 1 1
54035427
2 NULL 1
54045428
2 1 NULL
54055429

@@ -5411,6 +5435,7 @@ select array_ndims(column1), array_ndims(column2), array_ndims(column3) from lar
54115435
2 1 1
54125436
2 1 1
54135437
NULL 1 1
5438+
NULL 1 1
54145439
2 NULL 1
54155440
2 1 NULL
54165441

@@ -5732,6 +5757,7 @@ true false true false
57325757
true false false true
57335758
false true false false
57345759
NULL NULL false false
5760+
NULL NULL false false
57355761
false false NULL false
57365762
false false false NULL
57375763

@@ -5747,6 +5773,7 @@ true false true false
57475773
true false false true
57485774
false true false false
57495775
NULL NULL false false
5776+
NULL NULL false false
57505777
false false NULL false
57515778
false false false NULL
57525779

@@ -7068,27 +7095,29 @@ false
70687095
#NULL
70697096

70707097
# empty scalar function #5
7071-
query B
7072-
select empty(column1) from arrays;
7098+
query BB
7099+
select empty(column1), empty(column2) from arrays;
70737100
----
7074-
false
7075-
false
7076-
false
7077-
false
7078-
NULL
7079-
false
7080-
false
7101+
false false
7102+
false false
7103+
false false
7104+
false false
7105+
NULL false
7106+
NULL true
7107+
false NULL
7108+
false false
70817109

7082-
query B
7083-
select empty(arrow_cast(column1, 'LargeList(List(Int64))')) from arrays;
7110+
query BB
7111+
select empty(arrow_cast(column1, 'LargeList(List(Int64))')), empty(arrow_cast(column2, 'LargeList(Int64)')) from arrays;
70847112
----
7085-
false
7086-
false
7087-
false
7088-
false
7089-
NULL
7090-
false
7091-
false
7113+
false false
7114+
false false
7115+
false false
7116+
false false
7117+
NULL false
7118+
NULL true
7119+
false NULL
7120+
false false
70927121

70937122
query B
70947123
select empty(column1) from fixed_size_arrays;

0 commit comments

Comments
 (0)