feat(bigInt): support to 64 bit integers

Daniele Briggi · Daniele Briggi · commit 5ce0c23a0509 · 2025-05-20T08:03:03.000Z
diff --git a/.env.example b/.env.example
@@ -1,4 +1,7 @@
 
+# support to 64bit integers: number, bigint or mixed
+SAFE_INTEGER_MODE=number
+
 # chinook database used in non-destructive tests
 CHINOOK_DATABASE_URL="sqlitecloud://user:password@xxx.sqlite.cloud:8860/chinook.sqlite"
 
diff --git a/lib/drivers/protocol.d.ts b/lib/drivers/protocol.d.ts
@@ -1,5 +1,5 @@
-import { SQLiteCloudCommand, type SQLCloudRowsetMetadata, type SQLiteCloudDataTypes } from './types';
 import { SQLiteCloudRowset } from './rowset';
+import { SQLiteCloudCommand, type SQLCloudRowsetMetadata, type SQLiteCloudDataTypes } from './types';
 import { Buffer } from 'buffer';
 export declare const CMD_STRING = "+";
 export declare const CMD_ZEROSTRING = "!";
diff --git a/lib/drivers/protocol.js b/lib/drivers/protocol.js
@@ -15,8 +15,8 @@ exports.bufferEndsWith = bufferEndsWith;
 exports.parseRowsetChunks = parseRowsetChunks;
 exports.popData = popData;
 exports.formatCommand = formatCommand;
-const types_1 = require("./types");
 const rowset_1 = require("./rowset");
+const types_1 = require("./types");
 // explicitly importing buffer library to allow cross-platform support by replacing it
 const buffer_1 = require("buffer");
 // https://www.npmjs.com/package/lz4js
@@ -272,7 +272,17 @@ function popData(buffer) {
     // console.debug(`popData - dataType: ${dataType}, spaceIndex: ${spaceIndex}, commandLength: ${commandLength}, commandEnd: ${commandEnd}`)
     switch (dataType) {
         case exports.CMD_INT:
-            return popResults(parseInt(buffer.subarray(1, spaceIndex).toString()));
+            // SQLite uses 64-bit INTEGER, but JS uses 53-bit Number
+            const value = BigInt(buffer.subarray(1, spaceIndex).toString());
+            if (types_1.SAFE_INTEGER_MODE === 'bigint') {
+                return popResults(value);
+            }
+            if (types_1.SAFE_INTEGER_MODE === 'mixed') {
+                if (value <= BigInt(Number.MIN_SAFE_INTEGER) || BigInt(Number.MAX_SAFE_INTEGER) <= value) {
+                    return popResults(value);
+                }
+            }
+            return popResults(Number(value));
         case exports.CMD_FLOAT:
             return popResults(parseFloat(buffer.subarray(1, spaceIndex).toString()));
         case exports.CMD_NULL:
@@ -342,6 +352,9 @@ function serializeData(data, zeroString = false) {
             return buffer_1.Buffer.from(`${exports.CMD_FLOAT}${data} `);
         }
     }
+    if (typeof data === 'bigint') {
+        return buffer_1.Buffer.from(`${exports.CMD_INT}${data} `);
+    }
     if (buffer_1.Buffer.isBuffer(data)) {
         const header = `${exports.CMD_BLOB}${data.byteLength} `;
         return buffer_1.Buffer.concat([buffer_1.Buffer.from(header), data]);
diff --git a/lib/drivers/types.d.ts b/lib/drivers/types.d.ts
@@ -6,6 +6,16 @@ import tls from 'tls';
 export declare const DEFAULT_TIMEOUT: number;
 /** Default tls connection port */
 export declare const DEFAULT_PORT = 8860;
+/**
+ * Support to SQLite 64bit integer
+ *
+ * number - (default) always return Number type (max: 2^53 - 1)
+ *   Precision is lost when selecting greater numbers from SQLite
+ * bigint - always return BigInt type (max: 2^63 - 1) for all numbers from SQLite
+ *  (inlcuding `lastID` from WRITE statements)
+ * mixed - use BigInt and Number types depending on the value size
+ */
+export declare const SAFE_INTEGER_MODE: string | undefined;
 /**
  * Configuration for SQLite cloud connection
  * @note Options are all lowecase so they 1:1 compatible with C SDK
diff --git a/lib/drivers/types.js b/lib/drivers/types.js
@@ -2,12 +2,29 @@
 /**
  * types.ts - shared types and interfaces
  */
+var _a;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SQLiteCloudArrayType = exports.SQLiteCloudError = exports.DEFAULT_PORT = exports.DEFAULT_TIMEOUT = void 0;
+exports.SQLiteCloudArrayType = exports.SQLiteCloudError = exports.SAFE_INTEGER_MODE = exports.DEFAULT_PORT = exports.DEFAULT_TIMEOUT = void 0;
 /** Default timeout value for queries */
 exports.DEFAULT_TIMEOUT = 300 * 1000;
 /** Default tls connection port */
 exports.DEFAULT_PORT = 8860;
+/**
+ * Support to SQLite 64bit integer
+ *
+ * number - (default) always return Number type (max: 2^53 - 1)
+ *   Precision is lost when selecting greater numbers from SQLite
+ * bigint - always return BigInt type (max: 2^63 - 1) for all numbers from SQLite
+ *  (inlcuding `lastID` from WRITE statements)
+ * mixed - use BigInt and Number types depending on the value size
+ */
+exports.SAFE_INTEGER_MODE = (_a = process.env['SAFE_INTEGER_MODE']) === null || _a === void 0 ? void 0 : _a.toLowerCase();
+if (exports.SAFE_INTEGER_MODE == 'bigint') {
+    console.debug('BigInt mode: Using Number for all INTEGER values from SQLite, including meta information from WRITE statements.');
+}
+if (exports.SAFE_INTEGER_MODE == 'mixed') {
+    console.debug('Mixed mode: Using BigInt for INTEGER values from SQLite (including meta information from WRITE statements) bigger then 2^53, Number otherwise.');
+}
 /** Custom error reported by SQLiteCloud drivers */
 class SQLiteCloudError extends Error {
     constructor(message, args) {
diff --git a/lib/drivers/utilities.js b/lib/drivers/utilities.js
@@ -106,13 +106,12 @@ function getUpdateResults(results) {
             switch (results[0]) {
                 case types_1.SQLiteCloudArrayType.ARRAY_TYPE_SQLITE_EXEC:
                     return {
-                        type: results[0],
-                        index: results[1],
+                        type: Number(results[0]),
+                        index: Number(results[1]),
                         lastID: results[2], // ROWID (sqlite3_last_insert_rowid)
                         changes: results[3], // CHANGES(sqlite3_changes)
                         totalChanges: results[4], // TOTAL_CHANGES (sqlite3_total_changes)
-                        finalized: results[5], // FINALIZED
-                        //
+                        finalized: Number(results[5]), // FINALIZED
                         rowId: results[2] // same as lastId
                     };
             }
diff --git a/lib/sqlitecloud.drivers.dev.js b/lib/sqlitecloud.drivers.dev.js
diff --git a/lib/sqlitecloud.drivers.js b/lib/sqlitecloud.drivers.js
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@sqlitecloud/drivers",
-  "version": "1.0.494",
+  "version": "1.0.505",
   "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
diff --git a/src/drivers/protocol.ts b/src/drivers/protocol.ts
@@ -2,8 +2,8 @@
 // protocol.ts - low level protocol handling for SQLiteCloud transport
 //
 
-import { SQLiteCloudCommand, SQLiteCloudError, type SQLCloudRowsetMetadata, type SQLiteCloudDataTypes } from './types'
 import { SQLiteCloudRowset } from './rowset'
+import { SAFE_INTEGER_MODE, SQLiteCloudCommand, SQLiteCloudError, type SQLCloudRowsetMetadata, type SQLiteCloudDataTypes } from './types'
 
 // explicitly importing buffer library to allow cross-platform support by replacing it
 import { Buffer } from 'buffer'
@@ -302,7 +302,17 @@ export function popData(buffer: Buffer): { data: SQLiteCloudDataTypes | SQLiteCl
   // console.debug(`popData - dataType: ${dataType}, spaceIndex: ${spaceIndex}, commandLength: ${commandLength}, commandEnd: ${commandEnd}`)
   switch (dataType) {
     case CMD_INT:
-      return popResults(parseInt(buffer.subarray(1, spaceIndex).toString()))
+      // SQLite uses 64-bit INTEGER, but JS uses 53-bit Number
+      const value = BigInt(buffer.subarray(1, spaceIndex).toString())
+      if (SAFE_INTEGER_MODE === 'bigint') {
+        return popResults(value)
+      }
+      if (SAFE_INTEGER_MODE === 'mixed') {
+        if (value <= BigInt(Number.MIN_SAFE_INTEGER) || BigInt(Number.MAX_SAFE_INTEGER) <= value) {
+          return popResults(value)
+        }
+      }
+      return popResults(Number(value))
     case CMD_FLOAT:
       return popResults(parseFloat(buffer.subarray(1, spaceIndex).toString()))
     case CMD_NULL:
@@ -382,6 +392,10 @@ function serializeData(data: SQLiteCloudDataTypes, zeroString: boolean = false):
     }
   }
 
+  if (typeof data === 'bigint') {
+    return Buffer.from(`${CMD_INT}${data} `)
+  }
+
   if (Buffer.isBuffer(data)) {
     const header = `${CMD_BLOB}${data.byteLength} `
     return Buffer.concat([Buffer.from(header), data])
diff --git a/src/drivers/types.ts b/src/drivers/types.ts
@@ -10,6 +10,23 @@ export const DEFAULT_TIMEOUT = 300 * 1000
 /** Default tls connection port */
 export const DEFAULT_PORT = 8860
 
+/**
+ * Support to SQLite 64bit integer
+ *
+ * number - (default) always return Number type (max: 2^53 - 1)
+ *   Precision is lost when selecting greater numbers from SQLite
+ * bigint - always return BigInt type (max: 2^63 - 1) for all numbers from SQLite
+ *  (inlcuding `lastID` from WRITE statements)
+ * mixed - use BigInt and Number types depending on the value size
+ */
+export const SAFE_INTEGER_MODE = process.env['SAFE_INTEGER_MODE']?.toLowerCase()
+if (SAFE_INTEGER_MODE == 'bigint') {
+  console.debug('BigInt mode: Using Number for all INTEGER values from SQLite, including meta information from WRITE statements.')
+}
+if (SAFE_INTEGER_MODE == 'mixed') {
+  console.debug('Mixed mode: Using BigInt for INTEGER values from SQLite (including meta information from WRITE statements) bigger then 2^53, Number otherwise.')
+}
+
 /**
  * Configuration for SQLite cloud connection
  * @note Options are all lowecase so they 1:1 compatible with C SDK
diff --git a/src/drivers/utilities.ts b/src/drivers/utilities.ts
@@ -109,13 +109,12 @@ export function getUpdateResults(results?: any): Record<string, any> | undefined
       switch (results[0]) {
         case SQLiteCloudArrayType.ARRAY_TYPE_SQLITE_EXEC:
           return {
-            type: results[0],
-            index: results[1],
+            type: Number(results[0]),
+            index: Number(results[1]),
             lastID: results[2], // ROWID (sqlite3_last_insert_rowid)
             changes: results[3], // CHANGES(sqlite3_changes)
             totalChanges: results[4], // TOTAL_CHANGES (sqlite3_total_changes)
-            finalized: results[5], // FINALIZED
-            //
+            finalized: Number(results[5]), // FINALIZED
             rowId: results[2] // same as lastId
           }
       }
diff --git a/test/compare.test.ts b/test/compare.test.ts
@@ -2,12 +2,11 @@
  * compare.test.ts - test driver api against sqlite3 equivalents
  */
 
-import { CHINOOK_DATABASE_FILE, CHINOOK_FIRST_TRACK, LONG_TIMEOUT, removeDatabase, TESTING_SQL } from './shared'
-import { getChinookDatabase, getTestingDatabase } from './shared'
+import { CHINOOK_DATABASE_FILE, CHINOOK_FIRST_TRACK, getChinookDatabase, getTestingDatabase, LONG_TIMEOUT, removeDatabase, TESTING_SQL } from './shared'
 
 // https://github.com/TryGhost/node-sqlite3/wiki/API
-import sqlite3 from 'sqlite3'
 import { join } from 'path'
+import sqlite3 from 'sqlite3'
 
 const INSERT_SQL = "INSERT INTO people (name, hobby, age) VALUES ('Fantozzi Ugo', 'Competitive unicorn farting', 42); "
 const TESTING_DATABASE_FILE = join(__dirname, 'assets/testing.db')
@@ -252,6 +251,29 @@ describe('Database.get', () => {
     })
   })
 
+  it('should handle big integer (>= 2^53) as Number by default', done => {
+    const chinookCloud = getChinookDatabase()
+    const chinookFile = getChinookDatabaseFile()
+
+    const sql = 'SELECT 5876026397369593117 as bignumber;'
+    chinookCloud.get<any>(sql, (cloudError, cloudRow) => {
+      expect(cloudError).toBeFalsy()
+      expect(typeof (cloudRow as any)['bignumber']).toBe('number')
+      expect((cloudRow as any)['bignumber']).toBe(5876026397369593000)
+
+      chinookFile.get<any>(sql, (fileError, fileRow) => {
+        expect(fileError).toBeNull()
+        // node-sqlite3 does not support bigint: https://github.com/TryGhost/node-sqlite3/issues/922
+        expect(typeof (fileRow as any)['bignumber']).toBe('number')
+        expect((fileRow as any)['bignumber']).toBe(5876026397369593000)
+
+        chinookCloud.close()
+        chinookFile.close()
+        done()
+      })
+    })
+  })
+
   // end get
 })
 
diff --git a/test/connection-tls.test.ts b/test/connection-tls.test.ts
@@ -53,19 +53,19 @@ describe('connect', () => {
   })
 */
 
-it(
-  'should connect with config object string',
-  done => {
-    const configObj = getChinookConfig()
-    const connection = new SQLiteCloudTlsConnection(configObj, error => {
-      expect(error).toBeNull()
-      expect(connection.connected).toBe(true)
-      connection.close()
-      done()
-    })
-  },
-  LONG_TIMEOUT
-)
+  it(
+    'should connect with config object string',
+    done => {
+      const configObj = getChinookConfig()
+      const connection = new SQLiteCloudTlsConnection(configObj, error => {
+        expect(error).toBeNull()
+        expect(connection.connected).toBe(true)
+        connection.close()
+        done()
+      })
+    },
+    LONG_TIMEOUT
+  )
 
   it(
     'should connect with config object string and test command',
@@ -324,6 +324,23 @@ describe('send test commands', () => {
     LONG_TIMEOUT
   )
 
+  it('should use Number as default behavior for integer INT64', done => {
+    const chinook = getConnection()
+    chinook.sendCommands('TEST INT64', (error, results) => {
+      let err = null
+      try {
+        expect(error).toBeNull()
+        expect(typeof results).toBe('number')
+        expect(results).toBe(9223372036854776000)
+      } catch (error) {
+        err = error
+      } finally {
+        chinook.close()
+        err ? done(err) : done()
+      }
+    })
+  })
+
   it('should test null', done => {
     const connection = getConnection()
     connection.sendCommands('TEST NULL', (error, results) => {
diff --git a/test/database.test.ts b/test/database.test.ts
@@ -267,7 +267,7 @@ describe('Database.get', () => {
       })
     })
 
-    // call close() right after the execution 
+    // call close() right after the execution
     // of the query not in its callback
     chinook.close(error => {
       expect(error).toBeNull()
@@ -625,6 +625,25 @@ describe('Database.sql (async)', () => {
       await removeDatabaseAsync(database)
     }
   })
+
+  it('should insert bigInt value as a full 64 bit integer', async () => {
+    let database
+    try {
+      database = await getTestingDatabaseAsync()
+      const bigIntValue = BigInt(2 ** 63) - BigInt(1) // 9223372036854775807
+
+      await database.sql('CREATE TABLE IF NOT EXISTS bigints (id INTEGER PRIMARY KEY, value BIGINT NOT NULL);')
+      const meta = await database.sql('INSERT INTO bigints (value) VALUES (?);', bigIntValue)
+
+      // by default, BigInt values are returned as Number loosing precision
+      // retrieving it as text preserves the value
+      const results = await database.sql('SELECT CAST(value AS TEXT) as bigIntAsString FROM bigints WHERE id = ?;', meta.lastID)
+      expect(results).toHaveLength(1)
+      expect(results[0].bigIntAsString).toEqual(bigIntValue.toString())
+    } finally {
+      await removeDatabaseAsync(database)
+    }
+  })
 })
 
 it('should be connected', async () => {
diff --git a/test/protocol.test.ts b/test/protocol.test.ts
@@ -36,13 +36,13 @@ const testCases = [
   {
     query: 'SELECT ?, ?, ?, ?, ?',
     parameters: ['world', 123, 3.14, null, Buffer.from('hello')],
-    expected: '=57 6 !21 SELECT ?, ?, ?, ?, ?\x00!6 world\x00:123 ,3.14 _ $5 hello',
+    expected: '=57 6 !21 SELECT ?, ?, ?, ?, ?\x00!6 world\x00:123 ,3.14 _ $5 hello'
   },
   {
     query: 'SELECT ?',
     parameters: ["'hello world'"],
-    expected: "=32 2 !9 SELECT ?\x00!14 'hello world'\x00",
-  },
+    expected: "=32 2 !9 SELECT ?\x00!14 'hello world'\x00"
+  }
 ]
 
 describe('Format command', () => {

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@sqlitecloud/drivers",`
`3`		`- "version": "1.0.494",`
	`3`	`+ "version": "1.0.505",`
`4`	`4`	`"description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients",`
`5`	`5`	`"main": "./lib/index.js",`
`6`	`6`	`"types": "./lib/index.d.ts",`