bool type support, supervisor, dev/prod mods

Author: larsnovikov

.gitignore

@@ -9,4 +9,5 @@ src/.env
 src/vendor
 src/configs/*.json
 src/plugins/user/*.go
-src/plugins/user/*.so
+src/plugins/user/*.so
+src/horgh-replicator

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.10
 
 RUN apt-get update
-RUN apt-get -y install curl g++ make bzip2 nano unixodbc unixodbc-dev
+RUN apt-get -y install curl g++ make bzip2 nano supervisor unixodbc unixodbc-dev
 
 WORKDIR /go/src/horgh-replicator
 COPY . .
@@ -11,4 +11,13 @@ RUN tar -xvf /vertica-client.tar.gz -C /
 
 #installing dep and vendors
 RUN go get -u github.com/golang/dep/...
-CMD ["sh", "-c", "cd /go/src/horgh-replicator/src && dep ensure -update && /bin/bash"]
+
+# dev mode
+CMD ["sh", "-c", "cd /go/src/horgh-replicator/src && dep ensure -update && /bin/bash"]
+
+# prod mode
+# CMD ["sh", "-c", "cd /go/src/horgh-replicator/src \
+#    && dep ensure -update \
+#    && go build main.go \
+#    && mv main horgh-replicator \
+#    && /usr/bin/supervisord"]

README.md

@@ -58,3 +58,9 @@ Don't forget to set `binlog_do_db=<master_db_name>` and restart MySQL service.
 - Set start position of log: `go run main.go set-position <table> <name> <position>`
 - Loader for replication testing: `go run main.go load`
 
+### Container mode
+
+Where are 2 modes of docker container in Dockerfile:
+
+- Prod mode: build app and execute app as supervisor process
+- Dev mode: provides the opportunity for manual start and debug

docker-compose.yml

@@ -6,4 +6,5 @@ services:
       context: ./
     tty: true
     volumes:
-      - .:/go/src/horgh-replicator
+      - .:/go/src/horgh-replicator
+      - ./supervisor:/etc/supervisor/conf.d

examples/user.json

@@ -5,6 +5,7 @@
       "id",
       "name",
       "status",
+      "active",
       "created"
     ]
   },
@@ -32,6 +33,11 @@
           ]
         }
       },
+      {
+        "name": "active",
+        "key": false,
+        "mode": "bool"
+      },
       {
         "name": "created",
         "key": false,

sql/structure.sql

@@ -6,6 +6,7 @@ CREATE TABLE test.user
   id INT AUTO_INCREMENT PRIMARY KEY,
   name VARCHAR(40),
   status VARCHAR(255),
+  active bool,
   created TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL ON UPDATE CURRENT_TIMESTAMP
 ) engine=InnoDB;
 
@@ -23,6 +24,7 @@ CREATE TABLE test.user
   id INT AUTO_INCREMENT PRIMARY KEY,
   name VARCHAR(40),
   status VARCHAR(255),
+  active bool,
   created TIMESTAMP NOT NULL
 ) engine=InnoDB;
 
@@ -44,6 +46,7 @@ CREATE TABLE public."user"
   id BIGSERIAL PRIMARY KEY,
   name VARCHAR(40),
   status VARCHAR(255),
+  active bool,
   created TIMESTAMP DEFAULT NULL
 );
 
@@ -58,7 +61,7 @@ CREATE TABLE public.post
 /*For ClickHouse Slave*/
 CREATE DATABASE test;
 
-CREATE TABLE test.user (id Int32, name FixedString(40), status FixedString(255), created DateTime) ENGINE = MergeTree()
+CREATE TABLE test.user (id Int32, name FixedString(40), status FixedString(255), active UInt8, created DateTime) ENGINE = MergeTree()
 PARTITION BY id
 ORDER BY id;
 
@@ -73,6 +76,7 @@ CREATE TABLE "user"
   id INT,
   name VARCHAR,
   status VARCHAR,
+  active bool,
   created TIMESTAMP NOT NULL
 );
 

sql/test.sql

@@ -1,6 +1,6 @@
 /* This queries to test replication user table*/
-INSERT INTO test.user (`name`, `status`) VALUE ("Jack", "active");
-UPDATE test.user SET `name`='Tommy' ORDER BY RAND() LIMIT 1;
+INSERT INTO test.user (`name`, `status`, `active`) VALUE ("Jack", "active", false);
+UPDATE test.user SET `name`='Tommy', `status`=true ORDER BY RAND() LIMIT 1;
 DELETE FROM test.user LIMIT 1;
 
 /* This queries to test replication post table*/

src/parser/parser.go

@@ -68,7 +68,11 @@ func (m *BinlogParser) beforeSave(beforeSave connectors.ConfigBeforeSave, value
 func (m *BinlogParser) prepareType(fieldName string, fieldType string, value interface{}, params map[string]interface{}) {
 	switch fieldType {
 	case "bool":
-		params[fieldName] = value.(bool)
+		if value.(int8) == 1 {
+			params[fieldName] = true
+		} else {
+			params[fieldName] = false
+		}
 	case "int":
 		params[fieldName] = value.(int32)
 	case "string":
@@ -138,15 +142,6 @@ func (m *BinlogParser) floatHelper(e *canal.RowsEvent, n int, columnName string)
 	return float64(0)
 }
 
-func (m *BinlogParser) boolHelper(e *canal.RowsEvent, n int, columnName string) bool {
-
-	val := m.intHelper(e, n, columnName)
-	if val == 1 {
-		return true
-	}
-	return false
-}
-
 func (m *BinlogParser) stringHelper(e *canal.RowsEvent, n int, columnName string) string {
 
 	columnId := m.getBinlogIdByName(e, columnName)

src/tools/loader.go

@@ -14,7 +14,7 @@ const (
 	// goroutine count. WARNING if you set more 1, may be concurrency problems
 	ThreadCount = 1
 	// time to create queries in minutes
-	LoadTime = 5
+	LoadTime = 1
 )
 
 var CmdLoad = &cobra.Command{
@@ -63,8 +63,8 @@ func randInt(min int, max int) int {
 
 func makeQueries(id int) {
 	queries := []string{
-		"INSERT INTO test.user (`name`, `status`) VALUE ('Jack', 'active');",
-		"UPDATE test.user SET `name`='Tommy', status='dead' ORDER BY RAND() LIMIT 1",
+		"INSERT INTO test.user (`name`, `status`, `active`) VALUE ('Jack', 'active', false);",
+		"UPDATE test.user SET `name`='Tommy', status='dead', active=true ORDER BY RAND() LIMIT 1",
 		"DELETE FROM test.user ORDER BY RAND() LIMIT 1;",
 		"INSERT INTO test.post (`title`, `text`) VALUE ('Title', 'London is the capital of Great Britain');",
 		"UPDATE test.post SET title='New title' ORDER BY RAND() LIMIT 1;",

supervisor/bash.conf

@@ -0,0 +1,5 @@
+[supervisord]
+nodaemon=true
+
+[program:sshd]
+command=/bin/bash