SQLiteWriter only allow flat files (#68)

* SQLiteWriter only allow flat files

* Error in shape

* Test with and without sqlite writer

* Try this out

* Update datastation

* Fix go install/

* Try failing in ps1

Author: eatonphil

.github/workflows/pull_requests.yml

@@ -29,7 +29,10 @@ jobs:
 
       - run: ./scripts/ci/prepare_linux.sh
       - run: go build -trimpath -buildmode=pie -mod=readonly -modcacherw
-      - run: ./scripts/test.py
+      - name: Test defaults
+        run: ./scripts/test.py
+      - name: Test without SQLite writer
+        run: DSQ_NO_SQLITE_WRITER=true ./scripts/test.py
 
   dsq-tests-windows:
     runs-on: windows-latest
@@ -46,9 +49,11 @@ jobs:
         shell: bash
 
       # Subtle behavioral differences between powershell/cmd and bash. For example nested double quotes must be escaped.
-      - run: python3 ./scripts/test.py
+      - name: Test with powershell
+        run: python3 ./scripts/test.py
         shell: powershell
-      - run: python3 ./scripts/test.py
+      - name: Test with cmd.exe
+        run: python3 ./scripts/test.py
         shell: cmd
 
   dsq-tests-macos:

go.mod

@@ -8,7 +8,7 @@ go 1.18
 require (
 	github.com/chzyer/readline v1.5.0
 	github.com/google/uuid v1.3.0
-	github.com/multiprocessio/datastation/runner v0.0.0-20220614050412-e3776fe304bc
+	github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862
 	github.com/olekukonko/tablewriter v0.0.5
 )
 

go.sum

@@ -445,8 +445,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
-github.com/multiprocessio/datastation/runner v0.0.0-20220614050412-e3776fe304bc h1:PZrwLRdb13fjcblFznyzFjTZisKKLl15VTrN5Bp3EFw=
-github.com/multiprocessio/datastation/runner v0.0.0-20220614050412-e3776fe304bc/go.mod h1:UCms/xK08DspNqDDZ5XsaIlc39AuREmeELspFYghMGI=
+github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862 h1:bgzazkNqaktd7lQqOcG6TXG1iz8S1GeGbwEcy0/Zh9k=
+github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862/go.mod h1:UCms/xK08DspNqDDZ5XsaIlc39AuREmeELspFYghMGI=
 github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e h1:NlPl7amllnQyVAkZgjBvFEkKxJSba/R8ZpaTodc7SIQ=
 github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e/go.mod h1:huI4M/MrI5px/SgmXYi0a2byKikSLgDrnMQuXOqKtw4=
 github.com/multiprocessio/go-openoffice v0.0.0-20220110232726-064f5dda1956 h1:WVofL03Eq+z3LbDOfH5eKzu2U85LFZZngOMBlNaO/H0=

main.go

@@ -347,6 +347,11 @@ type args struct {
 
 func getArgs() (*args, error) {
 	args := &args{}
+
+	args.noSQLiteWriter = strings.ToLower(os.Getenv("DSQ_NO_SQLITE_WRITER")) == "true"
+	args.convertNumbers = strings.ToLower(os.Getenv("DSQ_CONVERT_NUMBERS")) == "true"
+	args.cacheSettings.Enabled = strings.ToLower(os.Getenv("DSQ_CACHE")) == "true"
+
 	osArgs := os.Args[1:]
 	for i := 0; i < len(osArgs); i++ {
 		arg := osArgs[i]
@@ -597,10 +602,7 @@ func _main() error {
 		}
 		mtm := runner.MimeType(mt)
 		useSQLiteWriter = useSQLiteWriter && (mtm == runner.CSVMimeType ||
-			mtm == runner.ParquetMimeType ||
-			mtm == runner.AvroMimeType ||
 			mtm == runner.TSVMimeType ||
-			mtm == runner.JSONLinesMimeType ||
 			mtm == runner.RegexpLinesMimeType)
 		if !useSQLiteWriter {
 			break

scripts/ci/prepare_windows.ps1

@@ -1,10 +1,24 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+$PSDefaultParameterValues['*:ErrorAction']='Stop'
+function ThrowOnNativeFailure {
+    if (-not $?)
+    {
+        throw 'Native Failure'
+    }
+}
+
 iwr -useb 'https://raw.githubusercontent.com/scoopinstaller/install/master/install.ps1' -outfile 'install.ps1'
 .\install.ps1 -RunAsAdmin
 Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH
-scoop install jq zip curl 7zip
+scoop install jq
+scoop install zip
+scoop install curl
+scoop install 7zip
+scoop install go@1.18.3
 
-curl -L -O "https://go.dev/dl/go1.18.windows-amd64.zip"
-unzip go1.18.windows-amd64.zip
-Join-Path $pwd "go\bin" >> $Env:GITHUB_PATH
+# curl -L -O "https://go.dev/dl/go1.18.windows-amd64.zip"
+# unzip go1.18.windows-amd64.zip
+# Join-Path $pwd "go\bin" >> $Env:GITHUB_PATH
 
 7z e testdata/taxi.csv.7z

scripts/test.py

@@ -34,6 +34,9 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon
         if arg == '-f' or arg == '--filter':
             if sys.argv[i+1].lower() not in name.lower():
                 return
+        if arg == '-fo' or arg == '--filter-out':
+            if sys.argv[i+1].lower() in name.lower():
+                return
 
     tests += 1
     skipped = True
@@ -297,13 +300,7 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon
 cat taxi.csv | ./dsq --cache -s csv 'SELECT passenger_count, COUNT(*), AVG(total_amount) FROM {} GROUP BY passenger_count ORDER BY COUNT(*) DESC'
 """
 
-test("Caching from pipe (second time so import not required sqlitewriter)", to_run, want, sort=True, winSkip=True, within_seconds=5)
-
-to_run = """
-cat taxi.csv | ./dsq --no-sqlite-writer --cache -s csv 'SELECT passenger_count, COUNT(*), AVG(total_amount) FROM {} GROUP BY passenger_count ORDER BY COUNT(*) DESC'
-"""
-
-test("Caching from pipe (second time so import not required, jsonwriter)", to_run, want, sort=True, winSkip=True, within_seconds=5)
+test("Caching from pipe (second time so import not required)", to_run, want, sort=True, winSkip=True, within_seconds=5)
 
 to_run = """
 cat testdata/taxi_trunc.csv | ./dsq --cache -s csv 'SELECT passenger_count, COUNT(*), AVG(total_amount) FROM {} GROUP BY passenger_count ORDER BY COUNT(*) DESC'"""
@@ -340,6 +337,10 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon
 want = '[{"a": 1, "b": 2, "c": "[1,2]"}]'
 test("https://github.com/multiprocessio/dsq/issues/36", to_run, want, sort=True)
 
+to_run = """./dsq ./testdata/regr/67.jsonl 'SELECT COUNT(1) AS count FROM {}'"""
+want = '[{"count": 1}]'
+test("https://github.com/multiprocessio/dsq/issues/67", to_run, want, sort=True)
+
 # END OF REGRESSION TESTS
 
 print(f"{tests - failures} of {tests} succeeded.")

sqlite.go

@@ -133,7 +133,7 @@ func (sw *SQLiteResultItemWriter) WriteRow(r any, written int) error {
 	for _, field := range sw.fields {
 		v := m[field]
 		switch t := v.(type) {
-		case map[string]any:
+		case []any:
 			bs, err := json.Marshal(t)
 			if err != nil {
 				return err

testdata/regr/67.jsonl

@@ -0,0 +1 @@
+{"foo":[]}