From 6478016bdb2ad302e6b94b5043fcfd59de1e74a4 Mon Sep 17 00:00:00 2001 From: Avinash Lingaloo <> Date: Tue, 26 Jan 2021 16:36:17 +0400 Subject: [PATCH 1/7] Android File Path [ COMPLETED ] Provides the user with the ability to pass in a file path instead of the database name only and creates the *.db file under a 'databases' sub-folder under the provided path. --- .idea/workspace.xml | 104 ++++++++++++++++++ .../java/com/nozbe/watermelondb/Database.kt | 15 ++- 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 000000000..92fb92998 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - true - - true - true - - - - - - - - - - - - - - - - - 1606915911502 - - - - - - - - - \ No newline at end of file From 8c19eb009e25ba2db9996d411a6683ebe46e2515 Mon Sep 17 00:00:00 2001 From: Avinash Lingaloo <> Date: Wed, 27 Jan 2021 14:53:20 +0400 Subject: [PATCH 3/7] Replaced .contains with .startsWith --- .../android/src/main/java/com/nozbe/watermelondb/Database.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/native/android/src/main/java/com/nozbe/watermelondb/Database.kt b/native/android/src/main/java/com/nozbe/watermelondb/Database.kt index de3f651c2..b3bf593dd 100644 --- a/native/android/src/main/java/com/nozbe/watermelondb/Database.kt +++ b/native/android/src/main/java/com/nozbe/watermelondb/Database.kt @@ -15,10 +15,10 @@ class Database(private val name: String, private val context: Context) { if (name == ":memory:" || name.contains("mode=memory")) { context.cacheDir.delete() File(context.cacheDir, name).path - } else if (name.contains("/") || name.contains("file")) { + } else if (name.startsWith("/") || name.startsWith("file")) { // Extracts the database name from the path val dbName = name.substringAfterLast("/") - + // Extracts the real path where the *.db file will be created val truePath = name.substringAfterLast("file://").substringBeforeLast("/") From 02688f76ed8afb6986d73d0a27e298ecf75ffa34 Mon Sep 17 00:00:00 2001 From: Avinash Lingaloo <> Date: Wed, 27 Jan 2021 16:27:40 +0400 Subject: [PATCH 4/7] Update CHANGELOG-Unreleased.md --- CHANGELOG-Unreleased.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-Unreleased.md b/CHANGELOG-Unreleased.md index a04511bd3..85c4ead31 100644 --- a/CHANGELOG-Unreleased.md +++ b/CHANGELOG-Unreleased.md @@ -1,7 +1,7 @@ # Changelog ## Unreleased - +- [Android] Provides the user with the ability to pass in a file path instead of the database name only and creates the *.db file under a 'databases' sub-folder under the provided path. Pattern for file path is *file:///data/user/0/com.mattermost.rnbeta/files/xxx.db* ### BREAKING CHANGES - [LokiJS] `useWebWorker` and `useIncrementalIndexedDB` options are now required (previously, skipping them would only trigger a warning) From 6f0d4f1ffc4cfe712195e50a6f6e3b38db789ecd Mon Sep 17 00:00:00 2001 From: Avinash Lingaloo Date: Thu, 7 Apr 2022 10:58:33 +0400 Subject: [PATCH 5/7] Merging master into fork (#2) * feat(TS): add unsafeExecute method * feat(TS): add sql migration step * [loki] improved diagnostic * add {sqlString:} option to sqliteAdapter.unsafeExecute * v0.24.1-2 * v0.24.1-3 * makeDispatcher error diagnostics * v0.24.1-4 * v0.24.1-5 Co-authored-by: Mickael Lecoq Co-authored-by: Radek Pietruszewski --- CHANGELOG-Unreleased.md | 1 + native/shared/Database.cpp | 6 +++--- native/shared/Database.h | 2 +- native/shared/DatabaseInstallation.cpp | 6 ++++++ package.json | 2 +- src/Schema/migrations/index.d.ts | 7 ++++++- src/adapters/lokijs/worker/DatabaseDriver.js | 4 +++- src/adapters/sqlite/index.js | 14 +++++++++----- src/adapters/sqlite/makeDispatcher/index.native.js | 6 +++++- src/adapters/sqlite/type.js | 1 + src/adapters/type.d.ts | 7 +++++++ src/adapters/type.js | 3 ++- 12 files changed, 45 insertions(+), 14 deletions(-) diff --git a/CHANGELOG-Unreleased.md b/CHANGELOG-Unreleased.md index 2c38c0071..e66526436 100644 --- a/CHANGELOG-Unreleased.md +++ b/CHANGELOG-Unreleased.md @@ -15,6 +15,7 @@ - [adapters] Adapter objects can now be distinguished by checking their `static adapterType` - [Query] New `Q.includes('foo')` query for case-sensitive exact string includes comparison - [adapters] Adapter objects now returns `dbName` +- [TypeScript] Add unsafeExecute method ### Performance diff --git a/native/shared/Database.cpp b/native/shared/Database.cpp index a2bd41005..1a1cabf43 100644 --- a/native/shared/Database.cpp +++ b/native/shared/Database.cpp @@ -20,9 +20,9 @@ Database::Database(jsi::Runtime *runtime, std::string path, bool usesExclusiveLo #ifdef ANDROID executeMultiple("pragma temp_store = memory;"); #endif - + executeMultiple("pragma journal_mode = WAL;"); - + #ifdef ANDROID // NOTE: This was added in an attempt to fix mysterious `database disk image is malformed` issue when using // headless JS services @@ -52,7 +52,7 @@ jsi::JSError Database::dbError(std::string description) { void Database::destroy() { const std::lock_guard lock(mutex_); - + if (isDestroyed_) { return; } diff --git a/native/shared/Database.h b/native/shared/Database.h index b852aa472..6c9bce496 100644 --- a/native/shared/Database.h +++ b/native/shared/Database.h @@ -31,6 +31,7 @@ class Database : public jsi::HostObject { jsi::Value unsafeLoadFromSync(int jsonId, jsi::Object &schema, std::string preamble, std::string postamble); void unsafeResetDatabase(jsi::String &schema, int schemaVersion); jsi::Value getLocal(jsi::String &key); + void executeMultiple(std::string sql); private: bool initialized_; @@ -53,7 +54,6 @@ class Database : public jsi::HostObject { void executeUpdate(std::string sql); void getRow(sqlite3_stmt *stmt); bool getNextRowOrTrue(sqlite3_stmt *stmt); - void executeMultiple(std::string sql); jsi::Object resultDictionary(sqlite3_stmt *statement); jsi::Array resultArray(sqlite3_stmt *statement); jsi::Array resultColumns(sqlite3_stmt *statement); diff --git a/native/shared/DatabaseInstallation.cpp b/native/shared/DatabaseInstallation.cpp index 820e305db..aa847e3d2 100644 --- a/native/shared/DatabaseInstallation.cpp +++ b/native/shared/DatabaseInstallation.cpp @@ -244,6 +244,12 @@ void Database::install(jsi::Runtime *runtime) { auto postamble = args[3].getString(rt).utf8(rt); return database->unsafeLoadFromSync(jsonId, schema, preamble, postamble); }); + createMethod(rt, adapter, "unsafeExecuteMultiple", 1, [database](jsi::Runtime &rt, const jsi::Value *args) { + assert(database->initialized_); + auto sqlString = args[0].getString(rt).utf8(rt); + database->executeMultiple(sqlString); + return jsi::Value::undefined(); + }); createMethod(rt, adapter, "unsafeResetDatabase", 2, [database](jsi::Runtime &rt, const jsi::Value *args) { assert(database->initialized_); jsi::String schema = args[0].getString(rt); diff --git a/package.json b/package.json index f8e94633a..7156cad63 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@nozbe/watermelondb", "description": "Build powerful React Native and React web apps that scale from hundreds to tens of thousands of records and remain fast", - "version": "0.24.1-1", + "version": "0.24.1-5", "scripts": { "build": "NODE_ENV=production node ./scripts/make.js", "dev": "NODE_ENV=development node ./scripts/make.js", diff --git a/src/Schema/migrations/index.d.ts b/src/Schema/migrations/index.d.ts index 295fa45dd..a36be7890 100644 --- a/src/Schema/migrations/index.d.ts +++ b/src/Schema/migrations/index.d.ts @@ -26,7 +26,12 @@ declare module '@nozbe/watermelondb/Schema/migrations' { columns: ColumnSchema[] } - export type MigrationStep = CreateTableMigrationStep | AddColumnsMigrationStep + export type SqlMigrationStep = { + type: 'sql' + sql: string + } + + export type MigrationStep = CreateTableMigrationStep | AddColumnsMigrationStep | SqlMigrationStep export interface Migration { toVersion: SchemaVersion diff --git a/src/adapters/lokijs/worker/DatabaseDriver.js b/src/adapters/lokijs/worker/DatabaseDriver.js index 5ea41ce96..f9731e575 100644 --- a/src/adapters/lokijs/worker/DatabaseDriver.js +++ b/src/adapters/lokijs/worker/DatabaseDriver.js @@ -362,7 +362,9 @@ export default class DatabaseDriver { await this.unsafeResetDatabase() } } else { - logger.warn('[Loki] Database has newer version than app schema. Resetting database.') + logger.warn( + `[Loki] Database has newer version ${dbVersion} than app schema ${schemaVersion}. Resetting database.`, + ) await this.unsafeResetDatabase() } } diff --git a/src/adapters/sqlite/index.js b/src/adapters/sqlite/index.js index d4bed70e6..80736fa82 100644 --- a/src/adapters/sqlite/index.js +++ b/src/adapters/sqlite/index.js @@ -347,13 +347,17 @@ export default class SQLiteAdapter implements DatabaseAdapter { operations && typeof operations === 'object' && Object.keys(operations).length === 1 && - Array.isArray(operations.sqls), - 'unsafeExecute expects an { sqls: [ [sql, [args..]], ... ] } object', + (Array.isArray(operations.sqls) || typeof operations.sqlString === 'string'), + "unsafeExecute expects an { sqls: [ [sql, [args..]], ... ] } or { sqlString: 'foo; bar' } object", ) } - const queries: SQLiteQuery[] = (operations: any).sqls - const batchOperations = queries.map(([sql, args]) => [IGNORE_CACHE, null, sql, [args]]) - this._dispatcher.call('batch', [batchOperations], callback) + if (operations.sqls) { + const queries: SQLiteQuery[] = (operations: any).sqls + const batchOperations = queries.map(([sql, args]) => [IGNORE_CACHE, null, sql, [args]]) + this._dispatcher.call('batch', [batchOperations], callback) + } else if (operations.sqlString) { + this._dispatcher.call('unsafeExecuteMultiple', [operations.sqlString], callback) + } } getLocal(key: string, callback: ResultCallback): void { diff --git a/src/adapters/sqlite/makeDispatcher/index.native.js b/src/adapters/sqlite/makeDispatcher/index.native.js index 341f2e7d5..0111633ed 100644 --- a/src/adapters/sqlite/makeDispatcher/index.native.js +++ b/src/adapters/sqlite/makeDispatcher/index.native.js @@ -65,7 +65,11 @@ class SqliteJsiDispatcher implements SqliteDispatcher { try { const method = this._db[methodName] if (!method) { - throw new Error('Cannot run database method because database failed to open') + throw new Error( + `Cannot run database method ${method} because database failed to open. ${Object.keys( + this._db, + ).join(',')}`, + ) } let result = method(...args) // On Android, errors are returned, not thrown - see DatabaseInstallation.cpp diff --git a/src/adapters/sqlite/type.js b/src/adapters/sqlite/type.js index 00275a9fc..0847b0740 100644 --- a/src/adapters/sqlite/type.js +++ b/src/adapters/sqlite/type.js @@ -72,6 +72,7 @@ export type SqliteDispatcherMethod = | 'provideSyncJson' | 'unsafeResetDatabase' | 'getLocal' + | 'unsafeExecuteMultiple' export interface SqliteDispatcher { call(methodName: SqliteDispatcherMethod, args: any[], callback: ResultCallback): void; diff --git a/src/adapters/type.d.ts b/src/adapters/type.d.ts index 9e3224a08..fe248c0be 100644 --- a/src/adapters/type.d.ts +++ b/src/adapters/type.d.ts @@ -1,3 +1,5 @@ +import { SQLiteQuery } from '@nozbe/watermelondb/adapters/sqlite' + declare module '@nozbe/watermelondb/adapters/type' { import { AppSchema, Model, Query, RawRecord, RecordId, TableName } from '@nozbe/watermelondb' @@ -9,6 +11,8 @@ declare module '@nozbe/watermelondb/adapters/type' { | ['markAsDeleted', Model] | ['destroyPermanently', Model] + export type UnsafeExecuteOperations = { sqls: SQLiteQuery[] } + export interface DatabaseAdapter { schema: AppSchema @@ -36,6 +40,9 @@ declare module '@nozbe/watermelondb/adapters/type' { // Destroys the whole database, its schema, indexes, everything. unsafeResetDatabase(): Promise + // Performs work on the underlying database - see concrete DatabaseAdapter implementation for more details + unsafeExecute(work: UnsafeExecuteOperations): Promise + // Fetches string value from local storage getLocal(key: string): Promise diff --git a/src/adapters/type.js b/src/adapters/type.js index 066f3219c..892954103 100644 --- a/src/adapters/type.js +++ b/src/adapters/type.js @@ -7,7 +7,7 @@ import type { RecordId } from '../Model' import type { RawRecord } from '../RawRecord' import type { ResultCallback } from '../utils/fp/Result' -import type { SQLiteQuery } from './sqlite/type' +import type { SQLiteQuery, SQL } from './sqlite/type' import type { Loki } from './lokijs/type' export type CachedFindResult = RecordId | ?RawRecord @@ -21,6 +21,7 @@ export type BatchOperation = export type UnsafeExecuteOperations = | $Exact<{ sqls: SQLiteQuery[] }> + | $Exact<{ sqlString: SQL }> // JSI-only | $Exact<{ loki: (Loki) => void }> export interface DatabaseAdapter { From 4ed6c417067f40c83884a264ed4722bd8dedc4e6 Mon Sep 17 00:00:00 2001 From: Avinash Lingaloo Date: Mon, 1 Aug 2022 23:08:44 +0400 Subject: [PATCH 6/7] Update Database.kt --- .../java/com/nozbe/watermelondb/Database.kt | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/native/android/src/main/java/com/nozbe/watermelondb/Database.kt b/native/android/src/main/java/com/nozbe/watermelondb/Database.kt index ca31e205a..ba86ee0bd 100644 --- a/native/android/src/main/java/com/nozbe/watermelondb/Database.kt +++ b/native/android/src/main/java/com/nozbe/watermelondb/Database.kt @@ -6,26 +6,47 @@ import android.database.sqlite.SQLiteCursor import android.database.sqlite.SQLiteCursorDriver import android.database.sqlite.SQLiteDatabase import android.database.sqlite.SQLiteQuery +import android.util.Log import java.io.File class Database( + private val name: String, private val context: Context, private val openFlags: Int = SQLiteDatabase.CREATE_IF_NECESSARY or SQLiteDatabase.ENABLE_WRITE_AHEAD_LOGGING ) { - + private val TAG = "XXX" private val db: SQLiteDatabase by lazy { // TODO: This SUCKS. Seems like Android doesn't like sqlite `?mode=memory&cache=shared` mode. To avoid random breakages, save the file to /tmp, but this is slow. // NOTE: This is because Android system SQLite is not compiled with SQLITE_USE_URI=1 // issue `PRAGMA cache=shared` query after connection when needed + + val path = if (name == ":memory:" || name.contains("mode=memory")) { context.cacheDir.delete() File(context.cacheDir, name).path - } else { + } + else if (name.startsWith("/") || name.startsWith("file")) { + // Extracts the database name from the path + val dbName = name.substringAfterLast("/") + if(dbName.contains(".db")){ + // Extracts the real path where the *.db file will be created + val directory = name.substringAfterLast("file://").substringBeforeLast("/") + + // Creates the directory + val fileObj = File(directory, "databases") + fileObj.mkdir() + File("${directory}/databases", dbName).path + }else{ + throw IllegalArgumentException("Database name should contain '.db' as extension") + } + } + else { // On some systems there is some kind of lock on `/databases` folder ¯\_(ツ)_/¯ context.getDatabasePath("$name.db").path.replace("/databases", "") } + return@lazy SQLiteDatabase.openDatabase(path, null, openFlags) } From 57a228154e7c9d551c49a3023286ad9ab4f5f6ab Mon Sep 17 00:00:00 2001 From: Avinash Lingaloo Date: Mon, 1 Aug 2022 23:10:46 +0400 Subject: [PATCH 7/7] Update Database.kt --- native/android/src/main/java/com/nozbe/watermelondb/Database.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/native/android/src/main/java/com/nozbe/watermelondb/Database.kt b/native/android/src/main/java/com/nozbe/watermelondb/Database.kt index ba86ee0bd..c7542d2d5 100644 --- a/native/android/src/main/java/com/nozbe/watermelondb/Database.kt +++ b/native/android/src/main/java/com/nozbe/watermelondb/Database.kt @@ -15,7 +15,6 @@ class Database( private val context: Context, private val openFlags: Int = SQLiteDatabase.CREATE_IF_NECESSARY or SQLiteDatabase.ENABLE_WRITE_AHEAD_LOGGING ) { - private val TAG = "XXX" private val db: SQLiteDatabase by lazy { // TODO: This SUCKS. Seems like Android doesn't like sqlite `?mode=memory&cache=shared` mode. To avoid random breakages, save the file to /tmp, but this is slow. // NOTE: This is because Android system SQLite is not compiled with SQLITE_USE_URI=1