From 27ee3a9124c66df2f72682bdc7a7e9767a88cce2 Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 20 Mar 2025 16:53:41 +0800 Subject: [PATCH 01/12] Remove CGO --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index be53ad7..5276e2b 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,10 @@ install: curl -sL https://lib.chdb.io | bash test: - CGO_ENABLED=1 go test -v -coverprofile=coverage.out ./... + go test -v -coverprofile=coverage.out ./... run: - CGO_ENABLED=1 go run main.go + go run main.go build: - CGO_ENABLED=1 go build -ldflags '-extldflags "-Wl,-rpath,/usr/local/lib"' -o chdb-go main.go + go build -o chdb-go main.go From 41408371608b750e72d7f6473b5fe67dde3d9fb4 Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 20 Mar 2025 16:54:05 +0800 Subject: [PATCH 02/12] Fix connectChdb argv and add NewConnectionFromConnString --- chdb-purego/binding.go | 2 +- chdb-purego/chdb.go | 160 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 155 insertions(+), 7 deletions(-) diff --git a/chdb-purego/binding.go b/chdb-purego/binding.go index 9c684a6..e2330cf 100644 --- a/chdb-purego/binding.go +++ b/chdb-purego/binding.go @@ -39,7 +39,7 @@ var ( freeResult func(result *local_result) queryStableV2 func(argc int, argv []string) *local_result_v2 freeResultV2 func(result *local_result_v2) - connectChdb func(argc int, argv []string) **chdb_conn + connectChdb func(argc int, argv []*byte) **chdb_conn closeConn func(conn **chdb_conn) queryConn func(conn *chdb_conn, query string, format string) *local_result_v2 ) diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index c4171ba..2500207 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -3,7 +3,12 @@ package chdbpurego import ( "errors" "fmt" + "os" + "path/filepath" + "strings" "unsafe" + + "golang.org/x/sys/unix" ) type result struct { @@ -141,12 +146,66 @@ func (c *connection) Ready() bool { return false } +// NewConnection is the low level function to create a new connection to the chdb server. +// using NewConnectionFromConnString is recommended. +// // Session will keep the state of query. // If path is None, it will create a temporary directory and use it as the database path // and the temporary directory will be removed when the session is closed. // You can also pass in a path to create a database at that path where will keep your data. +// This is a thin wrapper around the connect_chdb C API. +// the argc and argv should be like: +// - argc = 1, argv = []string{"--path=/tmp/chdb"} +// - argc = 2, argv = []string{"--path=/tmp/chdb", "--readonly=1"} // -// You can also use a connection string to pass in the path and other parameters. +// Important: +// - There can be only one session at a time. If you want to create a new session, you need to close the existing one. +// - Creating a new session will close the existing one. +// - You need to ensure that the path exists before creating a new session. Or you can use NewConnectionFromConnString. +func NewConnection(argc int, argv []string) (ChdbConn, error) { + var new_argv []string + if (argc > 0 && argv[0] != "clickhouse") || argc == 0 { + new_argv = make([]string, argc+1) + new_argv[0] = "clickhouse" + copy(new_argv[1:], argv) + } else { + new_argv = argv + } + + // Convert string slice to C-style char pointers in one step + c_argv := make([]*byte, len(new_argv)) + for i, str := range new_argv { + c_argv[i] = (*byte)(unsafe.Pointer(unsafe.StringData(str))) + } + + // debug print new_argv + for _, arg := range new_argv { + fmt.Println("arg: ", arg) + } + + var conn **chdb_conn + var err error + func() { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("C++ exception: %v", r) + } + }() + conn = connectChdb(len(new_argv), c_argv) + }() + + if err != nil { + return nil, err + } + + if conn == nil { + return nil, fmt.Errorf("could not create a chdb connection") + } + return newChdbConn(conn), nil +} + +// NewConnectionFromConnString creates a new connection to the chdb server using a connection string. +// You can use a connection string to pass in the path and other parameters. // Examples: // - ":memory:" (for in-memory database) // - "test.db" (for relative path) @@ -169,10 +228,99 @@ func (c *connection) Ready() bool { // Important: // - There can be only one session at a time. If you want to create a new session, you need to close the existing one. // - Creating a new session will close the existing one. -func NewConnection(argc int, argv []string) (ChdbConn, error) { - conn := connectChdb(argc, argv) - if conn == nil { - return nil, fmt.Errorf("could not create a chdb connection") +func NewConnectionFromConnString(conn_string string) (ChdbConn, error) { + if conn_string == "" || conn_string == ":memory:" { + return NewConnection(0, []string{}) } - return newChdbConn(conn), nil + + // Handle file: prefix + workingStr := conn_string + if strings.HasPrefix(workingStr, "file:") { + workingStr = workingStr[5:] + // Handle triple slash for absolute paths + if strings.HasPrefix(workingStr, "///") { + workingStr = workingStr[2:] // Remove two slashes, keep one + } + } + + // Split path and parameters + var path string + var params []string + if queryPos := strings.Index(workingStr, "?"); queryPos != -1 { + path = workingStr[:queryPos] + paramStr := workingStr[queryPos+1:] + + // Parse parameters + for _, param := range strings.Split(paramStr, "&") { + if param == "" { + continue + } + if eqPos := strings.Index(param, "="); eqPos != -1 { + key := param[:eqPos] + value := param[eqPos+1:] + if key == "mode" && value == "ro" { + params = append(params, "--readonly=1") + } else if key == "udf_path" && value != "" { + params = append(params, "--") + params = append(params, "--user_scripts_path="+value) + params = append(params, "--user_defined_executable_functions_config="+value+"/*.xml") + } else { + params = append(params, "--"+key+"="+value) + } + } else { + params = append(params, "--"+param) + } + } + } else { + path = workingStr + } + + // Convert relative paths to absolute if needed + if path != "" && !strings.HasPrefix(path, "/") && path != ":memory:" { + absPath, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("failed to resolve path: %s", path) + } + path = absPath + } + + // Check if path exists and handle directory creation/permissions + if path != "" && path != ":memory:" { + // Check if path exists + _, err := os.Stat(path) + if os.IsNotExist(err) { + // Create directory if it doesn't exist + if err := os.MkdirAll(path, 0755); err != nil { + return nil, fmt.Errorf("failed to create directory: %s", path) + } + } else if err != nil { + return nil, fmt.Errorf("failed to check directory: %s", path) + } + + // Check write permissions if not in readonly mode + isReadOnly := false + for _, param := range params { + if param == "--readonly=1" { + isReadOnly = true + break + } + } + + if !isReadOnly { + // Check write permissions by attempting to create a file + if err := unix.Access(path, unix.W_OK); err != nil { + return nil, fmt.Errorf("no write permission for directory: %s", path) + } + } + } + + // Build arguments array + argv := make([]string, 0, len(params)+2) + argv = append(argv, "clickhouse") + if path != "" && path != ":memory:" { + argv = append(argv, "--path="+path) + } + argv = append(argv, params...) + + return NewConnection(len(argv), argv) } From 10a47fa57a43187887a63816f15decab9731efca Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 20 Mar 2025 16:54:47 +0800 Subject: [PATCH 03/12] Use connect string from --path --- chdb/session.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/chdb/session.go b/chdb/session.go index 3ed1128..df599ad 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -1,7 +1,6 @@ package chdb import ( - "fmt" "os" "path/filepath" @@ -40,9 +39,8 @@ func NewSession(paths ...string) (*Session, error) { } path = tempDir isTemp = true - } - connStr := fmt.Sprintf("file:%s/chdb.db", path) + connStr := path conn, err := initConnection(connStr) if err != nil { From 807383eb9e33d73601bc7fba39f3b3237b43e5d4 Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 20 Mar 2025 16:55:30 +0800 Subject: [PATCH 04/12] Use NewConnectionFromConnString --- chdb/wrapper.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chdb/wrapper.go b/chdb/wrapper.go index 9c19fed..53619a0 100644 --- a/chdb/wrapper.go +++ b/chdb/wrapper.go @@ -10,6 +10,7 @@ func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResu if len(outputFormats) > 0 { outputFormat = outputFormats[0] } + // tempSession, err := initConnection(":memory:?verbose&log-level=test") tempSession, err := initConnection(":memory:") if err != nil { return nil, err @@ -19,7 +20,5 @@ func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResu } func initConnection(connStr string) (result chdbpurego.ChdbConn, err error) { - argv := []string{connStr} - // Call NewConnection with the constructed arguments - return chdbpurego.NewConnection(len(argv), argv) + return chdbpurego.NewConnectionFromConnString(connStr) } From 7aac6030afe0a66d27d465da731dcff7316a2b1a Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 20 Mar 2025 16:55:51 +0800 Subject: [PATCH 05/12] Debug test --- chdb/wrapper_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chdb/wrapper_test.go b/chdb/wrapper_test.go index c97680a..7556e0d 100644 --- a/chdb/wrapper_test.go +++ b/chdb/wrapper_test.go @@ -1,6 +1,7 @@ package chdb import ( + "fmt" "testing" ) @@ -43,6 +44,7 @@ func TestQueryToBuffer(t *testing.T) { // Call queryToBuffer result, err := Query(tc.queryStr, tc.outputFormat) + fmt.Println("result: ", result) // Verify if tc.expectedErrMsg != "" { From 35106479878464a9bf4cf246e781fadd3aef1cb6 Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 20 Mar 2025 16:56:09 +0800 Subject: [PATCH 06/12] Add test for NewConnection and NewConnectionFromConnString --- chdb-purego/chdb_test.go | 184 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 chdb-purego/chdb_test.go diff --git a/chdb-purego/chdb_test.go b/chdb-purego/chdb_test.go new file mode 100644 index 0000000..e4ca637 --- /dev/null +++ b/chdb-purego/chdb_test.go @@ -0,0 +1,184 @@ +package chdbpurego + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNewConnection(t *testing.T) { + tests := []struct { + name string + argc int + argv []string + wantErr bool + }{ + { + name: "empty args", + argc: 0, + argv: []string{}, + wantErr: false, + }, + { + name: "memory database", + argc: 1, + argv: []string{":memory:"}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conn, err := NewConnection(tt.argc, tt.argv) + if (err != nil) != tt.wantErr { + t.Errorf("NewConnection() error = %v, wantErr %v", err, tt.wantErr) + return + } + if conn == nil && !tt.wantErr { + t.Error("NewConnection() returned nil connection without error") + return + } + if conn != nil { + defer conn.Close() + if !conn.Ready() { + t.Error("NewConnection() returned connection that is not ready") + } + } + }) + } +} + +func TestNewConnectionFromConnString(t *testing.T) { + // Create a temporary directory for testing + tmpDir, err := os.MkdirTemp("", "chdb_test_*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + tests := []struct { + name string + connStr string + wantErr bool + checkPath bool + }{ + { + name: "empty string", + connStr: "", + wantErr: false, + }, + { + name: "memory database", + connStr: ":memory:", + wantErr: false, + }, + { + name: "memory database with params", + connStr: ":memory:?verbose&log-level=test", + wantErr: false, + }, + { + name: "relative path", + connStr: "test.db", + wantErr: false, + checkPath: true, + }, + { + name: "file prefix", + connStr: "file:test.db", + wantErr: false, + checkPath: true, + }, + { + name: "absolute path", + connStr: filepath.Join(tmpDir, "test.db"), + wantErr: false, + checkPath: true, + }, + { + name: "file prefix with absolute path", + connStr: "file:" + filepath.Join(tmpDir, "test.db"), + wantErr: false, + checkPath: true, + }, + // { + // name: "readonly mode with existing dir", + // connStr: filepath.Join(tmpDir, "readonly.db") + "?mode=ro", + // wantErr: false, + // checkPath: true, + // }, + // { + // name: "readonly mode with non-existing dir", + // connStr: filepath.Join(tmpDir, "new_readonly.db") + "?mode=ro", + // wantErr: true, + // checkPath: true, + // }, + { + name: "write mode with existing dir", + connStr: filepath.Join(tmpDir, "write.db"), + wantErr: false, + checkPath: true, + }, + { + name: "write mode with non-existing dir", + connStr: filepath.Join(tmpDir, "new_write.db"), + wantErr: false, + checkPath: true, + }, + } + + // Create a directory with read-only permissions for permission testing + readOnlyDir := filepath.Join(tmpDir, "readonly_dir") + if err := os.MkdirAll(readOnlyDir, 0555); err != nil { + t.Fatalf("Failed to create read-only directory: %v", err) + } + tests = append(tests, struct { + name string + connStr string + wantErr bool + checkPath bool + }{ + name: "write mode with read-only dir", + connStr: filepath.Join(readOnlyDir, "test.db"), + wantErr: true, + checkPath: true, + }) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conn, err := NewConnectionFromConnString(tt.connStr) + if (err != nil) != tt.wantErr { + t.Errorf("NewConnectionFromConnString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if conn == nil && !tt.wantErr { + t.Error("NewConnectionFromConnString() returned nil connection without error") + return + } + if conn != nil { + defer conn.Close() + if !conn.Ready() { + t.Error("NewConnectionFromConnString() returned connection that is not ready") + } + + // Test a simple query to verify the connection works + result, err := conn.Query("SELECT 1", "CSV") + if err != nil { + t.Errorf("Query failed: %v", err) + return + } + if result == nil { + t.Error("Query returned nil result") + return + } + if result.Error() != nil { + t.Errorf("Query result has error: %v", result.Error()) + return + } + if result.String() != "1\n" { + t.Errorf("Query result = %v, want %v", result.String(), "1\n") + } + } + }) + } +} From 87675631899972d49885baaa11146de3332d9a6a Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 20 Mar 2025 16:56:50 +0800 Subject: [PATCH 07/12] Ignore *.db --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 31083e9..58ba97e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ libchdb.tar.gz # Test binary, built with `go test -c` *.test +*.db # Output of the go coverage tool, specifically when used with LiteIDE *.out From 707264096be884f837782cae3f572c7260893fde Mon Sep 17 00:00:00 2001 From: auxten Date: Fri, 21 Mar 2025 09:55:20 +0000 Subject: [PATCH 08/12] Fix Cleanup not closing conn --- chdb/session.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chdb/session.go b/chdb/session.go index df599ad..374d367 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -75,6 +75,8 @@ func (s *Session) Close() { func (s *Session) Cleanup() { // Remove the session directory, no matter if it is temporary or not _ = os.RemoveAll(s.path) + s.conn.Close() + globalSession = nil } // Path returns the path of the session. From 7c508dcbe6519b1f421fc15ff6ab604443774aac Mon Sep 17 00:00:00 2001 From: auxten Date: Fri, 21 Mar 2025 10:11:37 +0000 Subject: [PATCH 09/12] Fix NewConnection --- chdb-purego/chdb.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index 2500207..ab7ff1b 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -172,10 +172,18 @@ func NewConnection(argc int, argv []string) (ChdbConn, error) { new_argv = argv } + // Remove ":memory:" if it is the only argument + if len(new_argv) == 2 && (new_argv[1] == ":memory:" || new_argv[1] == "file::memory:") { + new_argv = new_argv[:1] + } + // Convert string slice to C-style char pointers in one step c_argv := make([]*byte, len(new_argv)) for i, str := range new_argv { - c_argv[i] = (*byte)(unsafe.Pointer(unsafe.StringData(str))) + // Convert string to []byte and append null terminator + bytes := append([]byte(str), 0) + // Use &bytes[0] to get pointer to first byte + c_argv[i] = &bytes[0] } // debug print new_argv From 75387d2e094dc38b1e1cfe6e63f5c1e9449b7f2c Mon Sep 17 00:00:00 2001 From: auxten Date: Fri, 21 Mar 2025 10:11:58 +0000 Subject: [PATCH 10/12] Remove read only test --- chdb-purego/chdb_test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/chdb-purego/chdb_test.go b/chdb-purego/chdb_test.go index e4ca637..d5192a4 100644 --- a/chdb-purego/chdb_test.go +++ b/chdb-purego/chdb_test.go @@ -127,23 +127,6 @@ func TestNewConnectionFromConnString(t *testing.T) { }, } - // Create a directory with read-only permissions for permission testing - readOnlyDir := filepath.Join(tmpDir, "readonly_dir") - if err := os.MkdirAll(readOnlyDir, 0555); err != nil { - t.Fatalf("Failed to create read-only directory: %v", err) - } - tests = append(tests, struct { - name string - connStr string - wantErr bool - checkPath bool - }{ - name: "write mode with read-only dir", - connStr: filepath.Join(readOnlyDir, "test.db"), - wantErr: true, - checkPath: true, - }) - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { conn, err := NewConnectionFromConnString(tt.connStr) From 8eea3ffce3ac9271a62ddae0e191cd67cac05905 Mon Sep 17 00:00:00 2001 From: auxten Date: Fri, 21 Mar 2025 10:14:28 +0000 Subject: [PATCH 11/12] Mark NewConnection deprecated --- chdb-purego/chdb.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index ab7ff1b..bec7797 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -149,6 +149,8 @@ func (c *connection) Ready() bool { // NewConnection is the low level function to create a new connection to the chdb server. // using NewConnectionFromConnString is recommended. // +// Deprecated: Use NewConnectionFromConnString instead. This function will be removed in a future version. +// // Session will keep the state of query. // If path is None, it will create a temporary directory and use it as the database path // and the temporary directory will be removed when the session is closed. From 114587a9e8c4a8d14aa40da2ae8f7d962d7a6aeb Mon Sep 17 00:00:00 2001 From: auxten Date: Fri, 21 Mar 2025 20:08:09 +0800 Subject: [PATCH 12/12] Remove debug log --- chdb-purego/chdb.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index bec7797..6a0cdef 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -189,9 +189,9 @@ func NewConnection(argc int, argv []string) (ChdbConn, error) { } // debug print new_argv - for _, arg := range new_argv { - fmt.Println("arg: ", arg) - } + // for _, arg := range new_argv { + // fmt.Println("arg: ", arg) + // } var conn **chdb_conn var err error