Add simple user dictionary migrations (#25)

Author: subins2000

go.mod

@@ -1,5 +1,5 @@
 module github.com/varnamproject/govarnam
 
-go 1.15
+go 1.16
 
 require github.com/mattn/go-sqlite3 v1.14.6

govarnam/dictionary.go

@@ -8,14 +8,18 @@ package govarnam
 
 import (
 	"context"
-	sql "database/sql"
+	"embed"
 	"fmt"
+	"io/fs"
 	"log"
 	"os"
 	"path"
 	"time"
 )
 
+//go:embed migrations/*.sql
+var embedFS embed.FS
+
 // DictionaryResult result from dictionary search
 type DictionaryResult struct {
 	// Exactly found starting word matches.
@@ -54,108 +58,42 @@ func (varnam *Varnam) InitDict(dictPath string) error {
 	var err error
 
 	if !fileExists(dictPath) {
-		log.Printf("Making Varnam Learnings File at %s\n", dictPath)
-		os.MkdirAll(path.Dir(dictPath), 0750)
-
-		varnam.dictConn, err = makeDictionary(dictPath)
-	} else {
-		varnam.dictConn, err = openDB(dictPath)
+		log.Printf("Making Varnam Learnings Dir for %s\n", dictPath)
+		err := os.MkdirAll(path.Dir(dictPath), 0750)
+		if err != nil {
+			return err
+		}
 	}
 
-	// Since SQLite v3.12.0, default page size is 4096
-	varnam.dictConn.Exec("PRAGMA page_size=4096;")
-	// WAL makes writes & reads happen concurrently => significantly fast
-	varnam.dictConn.Exec("PRAGMA journal_mode=wal;")
+	varnam.dictConn, err = openDB(dictPath)
+	if err != nil {
+		return err
+	}
 
 	varnam.DictPath = dictPath
 
-	return err
-}
-
-func makeDictionary(dictPath string) (*sql.DB, error) {
-	conn, err := openDB(dictPath)
+	// cd into migrations directory
+	migrationsFS, err := fs.Sub(embedFS, "migrations")
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	queries := []string{
-		`
-		CREATE TABLE IF NOT EXISTS metadata (
-			key TEXT UNIQUE,
-			value TEXT
-		);
-		`,
-		`
-		CREATE TABLE IF NOT EXISTS words (
-			id INTEGER PRIMARY KEY,
-			word TEXT UNIQUE,
-			weight INTEGER DEFAULT 1,
-			learned_on INTEGER
-		);
-		`,
-		`
-		CREATE VIRTUAL TABLE IF NOT EXISTS words_fts USING FTS5(
-			word,
-			weight UNINDEXED,
-			learned_on UNINDEXED,
-			content='words',
-			content_rowid='id',
-			tokenize='ascii',
-			prefix='1 2',
-		);
-		`,
-		`
-		CREATE TRIGGER words_ai AFTER INSERT ON words
-			BEGIN
-				INSERT INTO words_fts (rowid, word)
-				VALUES (new.id, new.word);
-			END;
-		`,
-		`
-		CREATE TRIGGER words_ad AFTER DELETE ON words
-			BEGIN
-				INSERT INTO words_fts (words_fts, rowid, word)
-				VALUES ('delete', old.id, old.word);
-			END;
-		`,
-		`
-		CREATE TRIGGER words_au AFTER UPDATE ON words
-			BEGIN
-				INSERT INTO words_fts (words_fts, rowid, word)
-				VALUES ('delete', old.id, old.word);
-				INSERT INTO words_fts (rowid, word)
-				VALUES (new.id, new.word);
-			END;
-		`,
-		`
-		CREATE TABLE IF NOT EXISTS patterns (
-			pattern TEXT NOT NULL COLLATE NOCASE,
-			word_id INTEGER NOT NULL,
-			FOREIGN KEY(word_id) REFERENCES words(id) ON DELETE CASCADE,
-			PRIMARY KEY(pattern, word_id)
-		);
-		`}
-
-	// Note: FTS can't be applied on patterns because
-	// we require partial word search which FTS doesn't support
-
-	for _, query := range queries {
-		ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second)
-		defer cancelFunc()
-
-		stmt, err := conn.PrepareContext(ctx, query)
-		if err != nil {
-			return nil, err
-		}
-		defer stmt.Close()
+	mg, err := InitMigrate(varnam.dictConn, migrationsFS)
+	if err != nil {
+		return err
+	}
 
-		_, err = stmt.ExecContext(ctx)
-		if err != nil {
-			return nil, err
-		}
+	ranMigrations, err := mg.Run()
+	if ranMigrations != 0 {
+		log.Printf("ran %d migrations", ranMigrations)
 	}
 
-	return conn, nil
+	// Since SQLite v3.12.0, default page size is 4096
+	varnam.dictConn.Exec("PRAGMA page_size=4096;")
+	// WAL makes writes & reads happen concurrently => significantly fast
+	varnam.dictConn.Exec("PRAGMA journal_mode=wal;")
+
+	return err
 }
 
 type searchDictionaryType int32
@@ -372,42 +310,6 @@ func (varnam *Varnam) getMoreFromDictionary(ctx context.Context, words []Suggest
 	}
 }
 
-// A simpler function to get matches from pattern dictionary
-// Gets incomplete matches.
-// Eg: If pattern = "chin", will return "china"
-// TODO better function name ? Ambiguous ?
-func (varnam *Varnam) getTrailingFromPatternDictionary(ctx context.Context, pattern string) []Suggestion {
-	var results []Suggestion
-
-	select {
-	case <-ctx.Done():
-		return results
-	default:
-		rows, err := varnam.dictConn.QueryContext(ctx, "SELECT word, weight FROM words WHERE id IN (SELECT word_id FROM patterns WHERE pattern LIKE ?) ORDER BY weight DESC LIMIT 10", pattern+"%")
-
-		if err != nil {
-			log.Print(err)
-			return results
-		}
-
-		defer rows.Close()
-
-		for rows.Next() {
-			var item Suggestion
-			rows.Scan(&item.Word, &item.Weight)
-			item.Weight += VARNAM_LEARNT_WORD_MIN_WEIGHT
-			results = append(results, item)
-		}
-
-		err = rows.Err()
-		if err != nil {
-			log.Print(err)
-		}
-
-		return results
-	}
-}
-
 // Gets incomplete and complete matches from pattern dictionary
 // Eg: If pattern = "chin" or "chinayil", will return "china"
 func (varnam *Varnam) getFromPatternDictionary(ctx context.Context, pattern string) []PatternDictionarySuggestion {

govarnam/migrate.go

@@ -0,0 +1,116 @@
+package govarnam
+
+import (
+	sql "database/sql"
+	"io/fs"
+	"strings"
+)
+
+type migrate struct {
+	db *sql.DB
+	fs fs.FS
+}
+
+type migrationStatus struct {
+	lastRun       string
+	lastMigration string
+}
+
+func InitMigrate(db *sql.DB, fs fs.FS) (*migrate, error) {
+	_, err := db.Exec(`
+		CREATE TABLE IF NOT EXISTS migrations (
+			id INTEGER PRIMARY KEY,
+			name VARCHAR(200)
+		);
+	`)
+	if err != nil {
+		return nil, err
+	}
+
+	return &migrate{db, fs}, nil
+}
+
+func (mg *migrate) Status() (*migrationStatus, error) {
+	var lastRun string = ""
+	mg.db.QueryRow("SELECT name FROM migrations ORDER BY id DESC LIMIT 1").Scan(&lastRun)
+
+	files, err := fs.ReadDir(mg.fs, ".")
+	if err != nil {
+		return nil, err
+	}
+
+	lastMigration := files[len(files)-1].Name()
+
+	return &migrationStatus{lastRun, lastMigration}, nil
+}
+
+func (mg *migrate) Run() (int, error) {
+	ranMigrations := 0
+
+	status, err := mg.Status()
+	if err != nil {
+		return 0, err
+	}
+
+	if status.lastRun != status.lastMigration {
+		ranMigrations, err = mg.runMigrations(status)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	return ranMigrations, nil
+}
+
+func (mg *migrate) runMigrations(status *migrationStatus) (int, error) {
+	files, err := fs.ReadDir(mg.fs, ".")
+	if err != nil {
+		return 0, err
+	}
+
+	ranMigrations := 0
+
+	// lastRun will be empty if no migrations have been run
+	var foundLastRunMigration bool = (status.lastRun == "")
+
+	for _, file := range files {
+		fileNameParts := strings.Split(file.Name(), ".")
+		migrationName := fileNameParts[0]
+
+		// Run all migrations after the last ran migration
+		if !foundLastRunMigration {
+			foundLastRunMigration = (status.lastRun == migrationName)
+		} else {
+			fileContents, err := fs.ReadFile(mg.fs, file.Name())
+			if err != nil {
+				return 0, err
+			}
+
+			tx, err := mg.db.Begin()
+			if err != nil {
+				return 0, err
+			}
+
+			_, err = tx.Exec(string(fileContents))
+			if err != nil {
+				return 0, err
+			}
+
+			stmt, err := tx.Prepare("INSERT INTO migrations (name) VALUES(?)")
+			if err != nil {
+				tx.Rollback()
+				return 0, err
+			}
+			_, err = stmt.Exec(migrationName)
+			if err != nil {
+				tx.Rollback()
+			}
+
+			tx.Commit()
+
+			ranMigrations++
+		}
+	}
+
+	return ranMigrations, nil
+}

govarnam/migrate_test.go

@@ -0,0 +1,57 @@
+package govarnam
+
+import (
+	"database/sql"
+	"embed"
+	"io/fs"
+	"testing"
+)
+
+//go:embed testdata/*.sql
+var testdataFS embed.FS
+
+func TestMigration(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+
+	checkError(err)
+
+	testdataDirFS, err := fs.Sub(testdataFS, "testdata")
+	checkError(err)
+
+	dirFiles, err := fs.ReadDir(testdataDirFS, ".")
+	checkError(err)
+
+	mg, err := InitMigrate(db, testdataDirFS)
+	checkError(err)
+
+	_, err = db.Query("SELECT * FROM words")
+	assertEqual(t, err != nil, true)
+
+	ranMigrations, err := mg.Run()
+	assertEqual(t, err, nil)
+	assertEqual(t, ranMigrations, len(dirFiles))
+
+	_, err = db.Query("SELECT * FROM words")
+	assertEqual(t, err, nil)
+
+	// Part 2 : New Migrations
+
+	_, err = db.Query("SELECT * FROM words_fts")
+	assertEqual(t, err != nil, true)
+
+	migrationsFS, err := fs.Sub(embedFS, "migrations")
+	checkError(err)
+
+	dirFiles, err = fs.ReadDir(migrationsFS, ".")
+	checkError(err)
+
+	mg, err = InitMigrate(db, migrationsFS)
+	checkError(err)
+
+	ranMigrations, err = mg.Run()
+	assertEqual(t, err, nil)
+	assertEqual(t, ranMigrations, len(dirFiles))
+
+	_, err = db.Query("SELECT * FROM words_fts")
+	assertEqual(t, err, nil)
+}

govarnam/migrations/2021-04-10-init.sql

@@ -0,0 +1,18 @@
+CREATE TABLE IF NOT EXISTS metadata (
+  key TEXT UNIQUE,
+  value TEXT
+);
+
+CREATE TABLE IF NOT EXISTS words (
+  id INTEGER PRIMARY KEY,
+  word TEXT UNIQUE,
+  weight INTEGER DEFAULT 1,
+  learned_on INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS patterns (
+  pattern TEXT NOT NULL COLLATE NOCASE,
+  word_id INTEGER NOT NULL,
+  FOREIGN KEY(word_id) REFERENCES words(id) ON DELETE CASCADE,
+  PRIMARY KEY(pattern, word_id)
+);