64
64
#include < util/stream/str.h>
65
65
#include < util/stream/input.h>
66
66
#include < util/stream/file.h>
67
+ #include < util/string/type.h>
67
68
#include < util/system/execpath.h>
68
69
#include < util/system/guard.h>
69
70
#include < util/system/shellcommand.h>
@@ -214,46 +215,50 @@ TString DebugPath(NYT::TRichYPath path) {
214
215
return NYT::NodeToCanonicalYsonString (NYT::PathToNode (path), NYT::NYson::EYsonFormat::Text) + " (" + std::to_string (numColumns) + " columns)" ;
215
216
}
216
217
217
- void GetIntegerConstraints (const TExprNode::TPtr& column, bool & isSigned, ui64& minValueAbs, ui64& maxValueAbs) {
218
- EDataSlot toType = column->GetTypeAnn ()->Cast <TDataExprType>()->GetSlot ();
218
+ void GetIntegerConstraints (const TExprNode::TPtr& column, bool & isSigned, ui64& minValueAbs, ui64& maxValueAbs, bool & isOptional) {
219
+ const TDataExprType* dataType = nullptr ;
220
+ const bool columnHasDataType = IsDataOrOptionalOfData (column->GetTypeAnn (), isOptional, dataType);
221
+ YQL_ENSURE (columnHasDataType, " YtQLFilter: unsupported type of column " << column->Dump ());
222
+ YQL_ENSURE (dataType);
223
+ const EDataSlot dataSlot = dataType->Cast <TDataExprType>()->GetSlot ();
219
224
220
- // AllowIntegralConversion (may consider some refactoring)
221
- if (toType == EDataSlot::Uint8) {
225
+ // looks like AllowIntegralConversion (may consider some refactoring)
226
+ if (dataSlot == EDataSlot::Uint8) {
222
227
isSigned = false ;
223
228
minValueAbs = 0 ;
224
229
maxValueAbs = Max<ui8>();
225
230
}
226
- else if (toType == EDataSlot::Uint16) {
231
+ else if (dataSlot == EDataSlot::Uint16) {
227
232
isSigned = false ;
228
233
minValueAbs = 0 ;
229
234
maxValueAbs = Max<ui16>();
230
235
}
231
- else if (toType == EDataSlot::Uint32) {
236
+ else if (dataSlot == EDataSlot::Uint32) {
232
237
isSigned = false ;
233
238
minValueAbs = 0 ;
234
239
maxValueAbs = Max<ui32>();
235
240
}
236
- else if (toType == EDataSlot::Uint64) {
241
+ else if (dataSlot == EDataSlot::Uint64) {
237
242
isSigned = false ;
238
243
minValueAbs = 0 ;
239
244
maxValueAbs = Max<ui64>();
240
245
}
241
- else if (toType == EDataSlot::Int8) {
246
+ else if (dataSlot == EDataSlot::Int8) {
242
247
isSigned = true ;
243
248
minValueAbs = (ui64)Max<i8 >() + 1 ;
244
249
maxValueAbs = (ui64)Max<i8 >();
245
250
}
246
- else if (toType == EDataSlot::Int16) {
251
+ else if (dataSlot == EDataSlot::Int16) {
247
252
isSigned = true ;
248
253
minValueAbs = (ui64)Max<i16 >() + 1 ;
249
254
maxValueAbs = (ui64)Max<i16 >();
250
255
}
251
- else if (toType == EDataSlot::Int32) {
256
+ else if (dataSlot == EDataSlot::Int32) {
252
257
isSigned = true ;
253
258
minValueAbs = (ui64)Max<i32 >() + 1 ;
254
259
maxValueAbs = (ui64)Max<i32 >();
255
260
}
256
- else if (toType == EDataSlot::Int64) {
261
+ else if (dataSlot == EDataSlot::Int64) {
257
262
isSigned = true ;
258
263
minValueAbs = (ui64)Max<i64 >() + 1 ;
259
264
maxValueAbs = (ui64)Max<i64 >();
@@ -286,51 +291,102 @@ void ConvertComparisonForQL(const TStringBuf& opName, TStringBuilder& result) {
286
291
}
287
292
}
288
293
289
- void GenerateInputQueryIntegerComparison (const TStringBuf& opName, const TExprNode::TPtr& intColumn, const TExprNode::TPtr& intValue, TStringBuilder& result) {
294
+ void GenerateInputQueryIntegerComparison (const TStringBuf& opName, const TExprNode::TPtr& intColumn, const TExprNode::TPtr& intValue, const std::optional<bool >& nullValue, TStringBuilder& result) {
295
+ if (TMaybeNode<TCoNull>(intValue) || TMaybeNode<TCoNothing>(intValue)) {
296
+ YQL_ENSURE (nullValue.has_value (), " YtQLFilter: optional type without coalesce is not supported" );
297
+ if (nullValue.value ()) {
298
+ result << " TRUE" ;
299
+ } else {
300
+ result << " FALSE" ;
301
+ }
302
+ return ;
303
+ }
304
+
305
+ TMaybeNode<TCoIntegralCtor> maybeIntValue;
306
+ if (auto maybeJustValue = TMaybeNode<TCoJust>(intValue)) {
307
+ maybeIntValue = TMaybeNode<TCoIntegralCtor>(maybeJustValue.Cast ().Input ().Ptr ());
308
+ } else {
309
+ maybeIntValue = TMaybeNode<TCoIntegralCtor>(intValue);
310
+ }
311
+ YQL_ENSURE (maybeIntValue);
312
+
290
313
bool columnsIsSigned;
291
314
ui64 minValueAbs;
292
315
ui64 maxValueAbs;
293
- GetIntegerConstraints (intColumn, columnsIsSigned, minValueAbs, maxValueAbs);
316
+ bool columnIsOptional;
317
+ GetIntegerConstraints (intColumn, columnsIsSigned, minValueAbs, maxValueAbs, columnIsOptional);
318
+ YQL_ENSURE (!columnIsOptional || columnIsOptional && nullValue.has_value (), " YtQLFilter: optional type without coalesce is not supported" );
294
319
295
- const auto maybeInt = TMaybeNode<TCoIntegralCtor>(intValue);
296
- YQL_ENSURE (maybeInt);
297
320
bool hasSign;
298
321
bool isSigned;
299
322
ui64 valueAbs;
300
- ExtractIntegralValue (maybeInt .Ref (), false , hasSign, isSigned, valueAbs);
323
+ ExtractIntegralValue (maybeIntValue .Ref (), false , hasSign, isSigned, valueAbs);
301
324
325
+ std::optional<bool > constantFilter;
302
326
if (!hasSign && valueAbs > maxValueAbs) {
303
- // value is greater than maximum
327
+ // Value is greater than maximum.
304
328
if (opName == " >" || opName == " >=" || opName == " ==" ) {
305
- result << " FALSE " ;
329
+ constantFilter = false ;
306
330
} else {
307
- result << " TRUE " ;
331
+ constantFilter = true ;
308
332
}
309
333
} else if (hasSign && valueAbs > minValueAbs) {
310
- // value is less than minimum
334
+ // Value is less than minimum.
311
335
if (opName == " <" || opName == " <=" || opName == " ==" ) {
312
- result << " FALSE " ;
336
+ constantFilter = false ;
313
337
} else {
314
- result << " TRUE" ;
338
+ constantFilter = true ;
339
+ }
340
+ }
341
+
342
+ const auto columnName = intColumn->ChildPtr (1 )->Content ();
343
+ if (!constantFilter.has_value ()) {
344
+ // Value is in the range, comparison is not constant.
345
+ if (columnIsOptional) {
346
+ const bool isLess = opName == " <" || opName == " <=" ;
347
+ if (isLess && !nullValue.value ()) {
348
+ // QL will handle 'x [operation] NULL' as TRUE here, but we need FALSE.
349
+ QuoteColumnForQL (columnName, result);
350
+ result << " != NULL AND " ;
351
+ } else if (!isLess && nullValue.value ()) {
352
+ // QL will handle 'x [operation] NULL' as FALSE here, but we need TRUE.
353
+ QuoteColumnForQL (columnName, result);
354
+ result << " = NULL OR " ;
355
+ }
315
356
}
316
- } else {
317
- // value is in the range
318
- const auto columnName = intColumn->ChildPtr (1 )->Content ();
319
- const auto valueStr = maybeInt.Cast ().Literal ().Value ();
320
357
QuoteColumnForQL (columnName, result);
321
358
result << " " ;
322
359
ConvertComparisonForQL (opName, result);
360
+ const auto valueStr = maybeIntValue.Cast ().Literal ().Value ();
323
361
result << " " << valueStr;
362
+ } else if (constantFilter.value ()) {
363
+ // Value is out of the range, comparison is always TRUE.
364
+ if (columnIsOptional && !nullValue.value ()) {
365
+ // Handle comparison with NULL as FALSE.
366
+ QuoteColumnForQL (columnName, result);
367
+ result << " IS NOT NULL" ;
368
+ } else {
369
+ result << " TRUE" ;
370
+ }
371
+ } else {
372
+ // Value is out of the range, comparison is always FALSE.
373
+ if (columnIsOptional && nullValue.value ()) {
374
+ // Handle comparison with NULL as TRUE.
375
+ QuoteColumnForQL (columnName, result);
376
+ result << " IS NULL" ;
377
+ } else {
378
+ result << " FALSE" ;
379
+ }
324
380
}
325
381
}
326
382
327
- void GenerateInputQueryComparison (const TCoCompare& op, TStringBuilder& result) {
383
+ void GenerateInputQueryComparison (const TCoCompare& op, const std::optional< bool >& nullValue, TStringBuilder& result) {
328
384
YQL_ENSURE (op.Ref ().IsCallable ({" <" , " <=" , " >" , " >=" , " ==" , " !=" }));
329
385
const auto left = op.Left ().Ptr ();
330
386
const auto right = op.Right ().Ptr ();
331
387
332
388
if (left->IsCallable (" Member" )) {
333
- GenerateInputQueryIntegerComparison (op.CallableName (), left, right, result);
389
+ GenerateInputQueryIntegerComparison (op.CallableName (), left, right, nullValue, result);
334
390
} else {
335
391
YQL_ENSURE (right->IsCallable (" Member" ));
336
392
auto invertedOp = op.CallableName ();
@@ -343,17 +399,29 @@ void GenerateInputQueryComparison(const TCoCompare& op, TStringBuilder& result)
343
399
} else if (invertedOp == " >=" ) {
344
400
invertedOp = " <=" ;
345
401
}
346
- GenerateInputQueryIntegerComparison (invertedOp, right, left, result);
402
+ GenerateInputQueryIntegerComparison (invertedOp, right, left, nullValue, result);
347
403
}
348
404
}
349
405
350
406
void GenerateInputQueryWhereExpression (const TExprNode::TPtr& node, TStringBuilder& result) {
351
407
if (const auto maybeCompare = TMaybeNode<TCoCompare>(node)) {
352
- GenerateInputQueryComparison (maybeCompare.Cast (), result);
408
+ GenerateInputQueryComparison (maybeCompare.Cast (), {}, result);
353
409
} else if (node->IsCallable (" Not" )) {
354
- result << " NOT (" ;
410
+ const auto child = node->ChildPtr (0 );
411
+ if (child->IsCallable (" Exists" )) {
412
+ // Do not generate NOT (x IS NOT NULL).
413
+ result << " (" ;
414
+ GenerateInputQueryWhereExpression (child->ChildPtr (0 ), result);
415
+ result << " ) IS NULL" ;
416
+ } else {
417
+ result << " NOT (" ;
418
+ GenerateInputQueryWhereExpression (child, result);
419
+ result << " )" ;
420
+ }
421
+ } else if (node->IsCallable (" Exists" )) {
422
+ result << " (" ;
355
423
GenerateInputQueryWhereExpression (node->ChildPtr (0 ), result);
356
- result << " )" ;
424
+ result << " ) IS NOT NULL " ;
357
425
} else if (node->IsCallable ({" And" , " Or" })) {
358
426
const TStringBuf op = node->IsCallable (" And" ) ? " AND" : " OR" ;
359
427
@@ -367,6 +435,17 @@ void GenerateInputQueryWhereExpression(const TExprNode::TPtr& node, TStringBuild
367
435
GenerateInputQueryWhereExpression (node->Child (i), result);
368
436
result << " )" ;
369
437
};
438
+ } else if (node->IsCallable (" Coalesce" )) {
439
+ YQL_ENSURE (node->ChildrenSize () == 2 );
440
+ const auto op = TMaybeNode<TCoCompare>(node->Child (0 )).Cast ();
441
+ const auto nullValueStr = TMaybeNode<TCoBool>(node->Child (1 )).Cast ().Literal ().Value ();
442
+ const std::optional<bool > nullValue (IsTrue (nullValueStr));
443
+ GenerateInputQueryComparison (op, nullValue, result);
444
+ } else if (const auto maybeBool = TMaybeNode<TCoBool>(node)) {
445
+ result << maybeBool.Cast ().Literal ().Value ();
446
+ } else if (node->IsCallable (" Member" )) {
447
+ const auto columnName = node->ChildPtr (1 )->Content ();
448
+ QuoteColumnForQL (columnName, result);
370
449
} else {
371
450
YQL_ENSURE (false , " unexpected node type" );
372
451
}
0 commit comments