@@ -3,6 +3,7 @@ import * as path from 'path';
3
3
import stream from 'node:stream' ;
4
4
import util from 'node:util' ;
5
5
import { stringify , NIL } from 'uuid' ;
6
+ import Int64 from 'node-int64' ;
6
7
import fetch , { HeadersInit } from 'node-fetch' ;
7
8
import {
8
9
TSessionHandle ,
@@ -12,7 +13,6 @@ import {
12
13
TSparkArrowTypes ,
13
14
TSparkParameter ,
14
15
} from '../thrift/TCLIService_types' ;
15
- import { Int64 } from './hive/Types' ;
16
16
import IDBSQLSession , {
17
17
ExecuteStatementOptions ,
18
18
TypeInfoRequest ,
@@ -41,22 +41,35 @@ import IClientContext, { ClientConfig } from './contracts/IClientContext';
41
41
// Explicitly promisify a callback-style `pipeline` because `node:stream/promises` is not available in Node 14
42
42
const pipeline = util . promisify ( stream . pipeline ) ;
43
43
44
- const defaultMaxRows = 100000 ;
45
-
46
44
interface OperationResponseShape {
47
45
status : TStatus ;
48
46
operationHandle ?: TOperationHandle ;
49
47
directResults ?: TSparkDirectResults ;
50
48
}
51
49
52
- function getDirectResultsOptions ( maxRows : number | null = defaultMaxRows ) {
50
+ export function numberToInt64 ( value : number | bigint | Int64 ) : Int64 {
51
+ if ( value instanceof Int64 ) {
52
+ return value ;
53
+ }
54
+
55
+ if ( typeof value === 'bigint' ) {
56
+ const buffer = new ArrayBuffer ( BigInt64Array . BYTES_PER_ELEMENT ) ;
57
+ const view = new DataView ( buffer ) ;
58
+ view . setBigInt64 ( 0 , value , false ) ; // `false` to use big-endian order
59
+ return new Int64 ( Buffer . from ( buffer ) ) ;
60
+ }
61
+
62
+ return new Int64 ( value ) ;
63
+ }
64
+
65
+ function getDirectResultsOptions ( maxRows : number | bigint | Int64 | null | undefined , config : ClientConfig ) {
53
66
if ( maxRows === null ) {
54
67
return { } ;
55
68
}
56
69
57
70
return {
58
71
getDirectResults : {
59
- maxRows : new Int64 ( maxRows ) ,
72
+ maxRows : numberToInt64 ( maxRows ?? config . directResultsDefaultMaxRows ) ,
60
73
} ,
61
74
} ;
62
75
}
@@ -86,7 +99,6 @@ function getArrowOptions(config: ClientConfig): {
86
99
}
87
100
88
101
function getQueryParameters (
89
- sessionHandle : TSessionHandle ,
90
102
namedParameters ?: Record < string , DBSQLParameter | DBSQLParameterValue > ,
91
103
ordinalParameters ?: Array < DBSQLParameter | DBSQLParameterValue > ,
92
104
) : Array < TSparkParameter > {
@@ -184,12 +196,12 @@ export default class DBSQLSession implements IDBSQLSession {
184
196
const operationPromise = driver . executeStatement ( {
185
197
sessionHandle : this . sessionHandle ,
186
198
statement,
187
- queryTimeout : options . queryTimeout ,
199
+ queryTimeout : options . queryTimeout ? numberToInt64 ( options . queryTimeout ) : undefined ,
188
200
runAsync : true ,
189
- ...getDirectResultsOptions ( options . maxRows ) ,
201
+ ...getDirectResultsOptions ( options . maxRows , clientConfig ) ,
190
202
...getArrowOptions ( clientConfig ) ,
191
203
canDownloadResult : options . useCloudFetch ?? clientConfig . useCloudFetch ,
192
- parameters : getQueryParameters ( this . sessionHandle , options . namedParameters , options . ordinalParameters ) ,
204
+ parameters : getQueryParameters ( options . namedParameters , options . ordinalParameters ) ,
193
205
canDecompressLZ4Result : clientConfig . useLZ4Compression && Boolean ( LZ4 ) ,
194
206
} ) ;
195
207
const response = await this . handleResponse ( operationPromise ) ;
@@ -339,10 +351,11 @@ export default class DBSQLSession implements IDBSQLSession {
339
351
public async getTypeInfo ( request : TypeInfoRequest = { } ) : Promise < IOperation > {
340
352
await this . failIfClosed ( ) ;
341
353
const driver = await this . context . getDriver ( ) ;
354
+ const clientConfig = this . context . getConfig ( ) ;
342
355
const operationPromise = driver . getTypeInfo ( {
343
356
sessionHandle : this . sessionHandle ,
344
357
runAsync : true ,
345
- ...getDirectResultsOptions ( request . maxRows ) ,
358
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
346
359
} ) ;
347
360
const response = await this . handleResponse ( operationPromise ) ;
348
361
return this . createOperation ( response ) ;
@@ -357,10 +370,11 @@ export default class DBSQLSession implements IDBSQLSession {
357
370
public async getCatalogs ( request : CatalogsRequest = { } ) : Promise < IOperation > {
358
371
await this . failIfClosed ( ) ;
359
372
const driver = await this . context . getDriver ( ) ;
373
+ const clientConfig = this . context . getConfig ( ) ;
360
374
const operationPromise = driver . getCatalogs ( {
361
375
sessionHandle : this . sessionHandle ,
362
376
runAsync : true ,
363
- ...getDirectResultsOptions ( request . maxRows ) ,
377
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
364
378
} ) ;
365
379
const response = await this . handleResponse ( operationPromise ) ;
366
380
return this . createOperation ( response ) ;
@@ -375,12 +389,13 @@ export default class DBSQLSession implements IDBSQLSession {
375
389
public async getSchemas ( request : SchemasRequest = { } ) : Promise < IOperation > {
376
390
await this . failIfClosed ( ) ;
377
391
const driver = await this . context . getDriver ( ) ;
392
+ const clientConfig = this . context . getConfig ( ) ;
378
393
const operationPromise = driver . getSchemas ( {
379
394
sessionHandle : this . sessionHandle ,
380
395
catalogName : request . catalogName ,
381
396
schemaName : request . schemaName ,
382
397
runAsync : true ,
383
- ...getDirectResultsOptions ( request . maxRows ) ,
398
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
384
399
} ) ;
385
400
const response = await this . handleResponse ( operationPromise ) ;
386
401
return this . createOperation ( response ) ;
@@ -395,14 +410,15 @@ export default class DBSQLSession implements IDBSQLSession {
395
410
public async getTables ( request : TablesRequest = { } ) : Promise < IOperation > {
396
411
await this . failIfClosed ( ) ;
397
412
const driver = await this . context . getDriver ( ) ;
413
+ const clientConfig = this . context . getConfig ( ) ;
398
414
const operationPromise = driver . getTables ( {
399
415
sessionHandle : this . sessionHandle ,
400
416
catalogName : request . catalogName ,
401
417
schemaName : request . schemaName ,
402
418
tableName : request . tableName ,
403
419
tableTypes : request . tableTypes ,
404
420
runAsync : true ,
405
- ...getDirectResultsOptions ( request . maxRows ) ,
421
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
406
422
} ) ;
407
423
const response = await this . handleResponse ( operationPromise ) ;
408
424
return this . createOperation ( response ) ;
@@ -417,10 +433,11 @@ export default class DBSQLSession implements IDBSQLSession {
417
433
public async getTableTypes ( request : TableTypesRequest = { } ) : Promise < IOperation > {
418
434
await this . failIfClosed ( ) ;
419
435
const driver = await this . context . getDriver ( ) ;
436
+ const clientConfig = this . context . getConfig ( ) ;
420
437
const operationPromise = driver . getTableTypes ( {
421
438
sessionHandle : this . sessionHandle ,
422
439
runAsync : true ,
423
- ...getDirectResultsOptions ( request . maxRows ) ,
440
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
424
441
} ) ;
425
442
const response = await this . handleResponse ( operationPromise ) ;
426
443
return this . createOperation ( response ) ;
@@ -435,14 +452,15 @@ export default class DBSQLSession implements IDBSQLSession {
435
452
public async getColumns ( request : ColumnsRequest = { } ) : Promise < IOperation > {
436
453
await this . failIfClosed ( ) ;
437
454
const driver = await this . context . getDriver ( ) ;
455
+ const clientConfig = this . context . getConfig ( ) ;
438
456
const operationPromise = driver . getColumns ( {
439
457
sessionHandle : this . sessionHandle ,
440
458
catalogName : request . catalogName ,
441
459
schemaName : request . schemaName ,
442
460
tableName : request . tableName ,
443
461
columnName : request . columnName ,
444
462
runAsync : true ,
445
- ...getDirectResultsOptions ( request . maxRows ) ,
463
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
446
464
} ) ;
447
465
const response = await this . handleResponse ( operationPromise ) ;
448
466
return this . createOperation ( response ) ;
@@ -457,13 +475,14 @@ export default class DBSQLSession implements IDBSQLSession {
457
475
public async getFunctions ( request : FunctionsRequest ) : Promise < IOperation > {
458
476
await this . failIfClosed ( ) ;
459
477
const driver = await this . context . getDriver ( ) ;
478
+ const clientConfig = this . context . getConfig ( ) ;
460
479
const operationPromise = driver . getFunctions ( {
461
480
sessionHandle : this . sessionHandle ,
462
481
catalogName : request . catalogName ,
463
482
schemaName : request . schemaName ,
464
483
functionName : request . functionName ,
465
484
runAsync : true ,
466
- ...getDirectResultsOptions ( request . maxRows ) ,
485
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
467
486
} ) ;
468
487
const response = await this . handleResponse ( operationPromise ) ;
469
488
return this . createOperation ( response ) ;
@@ -472,13 +491,14 @@ export default class DBSQLSession implements IDBSQLSession {
472
491
public async getPrimaryKeys ( request : PrimaryKeysRequest ) : Promise < IOperation > {
473
492
await this . failIfClosed ( ) ;
474
493
const driver = await this . context . getDriver ( ) ;
494
+ const clientConfig = this . context . getConfig ( ) ;
475
495
const operationPromise = driver . getPrimaryKeys ( {
476
496
sessionHandle : this . sessionHandle ,
477
497
catalogName : request . catalogName ,
478
498
schemaName : request . schemaName ,
479
499
tableName : request . tableName ,
480
500
runAsync : true ,
481
- ...getDirectResultsOptions ( request . maxRows ) ,
501
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
482
502
} ) ;
483
503
const response = await this . handleResponse ( operationPromise ) ;
484
504
return this . createOperation ( response ) ;
@@ -493,6 +513,7 @@ export default class DBSQLSession implements IDBSQLSession {
493
513
public async getCrossReference ( request : CrossReferenceRequest ) : Promise < IOperation > {
494
514
await this . failIfClosed ( ) ;
495
515
const driver = await this . context . getDriver ( ) ;
516
+ const clientConfig = this . context . getConfig ( ) ;
496
517
const operationPromise = driver . getCrossReference ( {
497
518
sessionHandle : this . sessionHandle ,
498
519
parentCatalogName : request . parentCatalogName ,
@@ -502,7 +523,7 @@ export default class DBSQLSession implements IDBSQLSession {
502
523
foreignSchemaName : request . foreignSchemaName ,
503
524
foreignTableName : request . foreignTableName ,
504
525
runAsync : true ,
505
- ...getDirectResultsOptions ( request . maxRows ) ,
526
+ ...getDirectResultsOptions ( request . maxRows , clientConfig ) ,
506
527
} ) ;
507
528
const response = await this . handleResponse ( operationPromise ) ;
508
529
return this . createOperation ( response ) ;
0 commit comments