Skip to content

Commit c1331ff

Browse files
ashish10alexashish10alex
committed
feat: sqlx parser to support multiple pre/post operation blocks
1 parent d5035d2 commit c1331ff

File tree

2 files changed

+234
-22
lines changed

2 files changed

+234
-22
lines changed

‎cmd/sqlx_parser.go

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
package cmd
2+
3+
import (
4+
"bufio"
5+
"fmt"
6+
"os"
7+
"strings"
8+
)
9+
10+
type ConfigBlockMeta struct {
11+
exsists bool
12+
startOfConfigBlock int
13+
endOfConfigBlock int
14+
configBlockContent string
15+
}
16+
17+
type PreOpsBlockMeta struct {
18+
exsists bool
19+
startOfPreOperationsBlock int
20+
endOfPreOperationsBlock int
21+
preOpsBlockContent string
22+
}
23+
24+
type PostOpsBlockMeta struct {
25+
exsists bool
26+
startOfpostOperationsBlock int
27+
endOfpostOperationsBlock int
28+
postOpsBlockContent string
29+
}
30+
31+
type SqlBlockMeta struct {
32+
exsists bool
33+
startOfSqlBlock int
34+
endOfSqlBlock int
35+
sqlBlockContent string
36+
formattedSqlBlockContent string
37+
}
38+
39+
type sqlxParserMeta struct {
40+
filepath string
41+
numLines int
42+
configBlockMeta ConfigBlockMeta
43+
preOpsBlocksMeta []PreOpsBlockMeta
44+
postOpsBlocksMeta []PostOpsBlockMeta
45+
sqlBlocksMeta SqlBlockMeta
46+
}
47+
48+
func sqlxParser(filepath string) (sqlxParserMeta, error) {
49+
50+
var inMajorBlock = false
51+
52+
var startOfConfigBlock = 0
53+
var endOfConfigBlock = 0
54+
var configBlockExsists = false
55+
var configBlockContent = ""
56+
57+
var preOpsBlocksMeta = []PreOpsBlockMeta{}
58+
var startOfPreOperationsBlock = 0
59+
var endOfPreOperationsBlock = 0
60+
61+
var postOpsBlocksMeta = []PostOpsBlockMeta{}
62+
var startOfpostOperationsBlock = 0
63+
var endOfpostOperationsBlock = 0
64+
65+
var startOfSqlBlock = 0
66+
var endOfSqlBlock = 0
67+
var sqlBlockExsists = false
68+
var sqlBlockContent = ""
69+
70+
var isInInnerMajorBlock = false
71+
var innerMajorBlockCount = 0
72+
73+
var currentBlock = ""
74+
var currentBlockContent = ""
75+
76+
file, err := os.Open(filepath)
77+
if err != nil {
78+
fmt.Printf("Error: %v\n", err)
79+
return sqlxParserMeta{}, err
80+
}
81+
82+
i := 0
83+
84+
scanner := bufio.NewScanner(file)
85+
for scanner.Scan() {
86+
i++
87+
var lineContents = scanner.Text() + "\n"
88+
89+
if strings.Contains(lineContents, "config {") {
90+
inMajorBlock = true
91+
currentBlock = "config"
92+
startOfConfigBlock = i
93+
currentBlockContent += lineContents
94+
} else if strings.Contains(lineContents, "post_operations {") && !inMajorBlock {
95+
startOfpostOperationsBlock = i
96+
inMajorBlock = true
97+
currentBlock = "post_operations"
98+
currentBlockContent += lineContents
99+
} else if strings.Contains(lineContents, "pre_operations {") && !inMajorBlock {
100+
inMajorBlock = true
101+
currentBlock = "pre_operations"
102+
startOfPreOperationsBlock = i
103+
currentBlockContent += lineContents
104+
} else if strings.Contains(lineContents, "{") && inMajorBlock {
105+
if strings.Contains(lineContents, "}") {
106+
continue
107+
}
108+
isInInnerMajorBlock = true
109+
innerMajorBlockCount += 1
110+
currentBlockContent += lineContents
111+
} else if strings.Contains(lineContents, "}") && isInInnerMajorBlock && innerMajorBlockCount >= 1 && inMajorBlock {
112+
innerMajorBlockCount -= 1
113+
currentBlockContent += lineContents
114+
} else if strings.Contains(lineContents, "}") && innerMajorBlockCount == 0 && inMajorBlock {
115+
if currentBlock == "config" {
116+
currentBlockContent += lineContents
117+
configBlockContent = currentBlockContent
118+
endOfConfigBlock = i
119+
configBlockExsists = true
120+
currentBlock = ""
121+
currentBlockContent = ""
122+
} else if currentBlock == "pre_operations" {
123+
endOfPreOperationsBlock = i
124+
currentBlockContent += lineContents
125+
preOpsBlockMeta := PreOpsBlockMeta{
126+
exsists: true,
127+
startOfPreOperationsBlock: startOfPreOperationsBlock,
128+
endOfPreOperationsBlock: endOfPreOperationsBlock,
129+
preOpsBlockContent: currentBlockContent,
130+
}
131+
preOpsBlocksMeta = append(preOpsBlocksMeta, preOpsBlockMeta)
132+
currentBlock = ""
133+
currentBlockContent = ""
134+
} else if currentBlock == "post_operations" {
135+
endOfpostOperationsBlock = i
136+
currentBlockContent += lineContents
137+
postOpsBlockMeta := PostOpsBlockMeta{
138+
exsists: true,
139+
startOfpostOperationsBlock: startOfpostOperationsBlock,
140+
endOfpostOperationsBlock: endOfpostOperationsBlock,
141+
postOpsBlockContent: currentBlockContent,
142+
}
143+
postOpsBlocksMeta = append(postOpsBlocksMeta, postOpsBlockMeta)
144+
currentBlock = ""
145+
currentBlockContent = ""
146+
}
147+
inMajorBlock = false
148+
} else if strings.Contains(lineContents, "}") && isInInnerMajorBlock && innerMajorBlockCount >= 1 && !inMajorBlock {
149+
innerMajorBlockCount -= 1
150+
currentBlockContent += lineContents
151+
} else if lineContents != "" && !inMajorBlock {
152+
if startOfSqlBlock == 0 {
153+
startOfSqlBlock = i
154+
sqlBlockExsists = true
155+
sqlBlockContent += lineContents
156+
} else {
157+
sqlBlockContent += lineContents
158+
endOfSqlBlock = i
159+
}
160+
} else if inMajorBlock {
161+
currentBlockContent += lineContents
162+
}
163+
}
164+
165+
// fmt.Println("configBlockContent: ", configBlockContent)
166+
// fmt.Println("preOpsBlockContent: ", postOpsBlocksMeta[0].postOpsBlockContent)
167+
// fmt.Println("postOpsBlockContent: ", postOpsBlocksMeta[0].postOpsBlockContent)
168+
// fmt.Println("sqlBlockContent: ", sqlBlockContent)
169+
170+
return sqlxParserMeta{
171+
filepath: filepath,
172+
numLines: i,
173+
configBlockMeta: ConfigBlockMeta{
174+
exsists: configBlockExsists,
175+
startOfConfigBlock: startOfConfigBlock,
176+
endOfConfigBlock: endOfConfigBlock,
177+
configBlockContent: configBlockContent,
178+
},
179+
preOpsBlocksMeta: preOpsBlocksMeta,
180+
postOpsBlocksMeta: postOpsBlocksMeta,
181+
sqlBlocksMeta: SqlBlockMeta{
182+
exsists: sqlBlockExsists,
183+
startOfSqlBlock: startOfSqlBlock,
184+
endOfSqlBlock: endOfSqlBlock,
185+
sqlBlockContent: sqlBlockContent,
186+
formattedSqlBlockContent: "",
187+
},
188+
}, nil
189+
}

‎cmd/utils.go

Lines changed: 45 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ func findSqlxFiles(dataformRootDirectory string) *[]string {
4848
return nil
4949
}
5050

51-
func formatSqlCode(sqlxFileMetaData *sqlxFileMetaData, pythonScriptPath string, sqlfluffConfigPath string, pythonExecutable string, logger *slog.Logger) error {
52-
queryString := *&sqlxFileMetaData.queryString
51+
func formatSqlCode(sqlxFileMetaData *sqlxParserMeta, pythonScriptPath string, sqlfluffConfigPath string, pythonExecutable string, logger *slog.Logger) error {
52+
queryString := *&sqlxFileMetaData.sqlBlocksMeta.sqlBlockContent
5353

5454
cmd := exec.Command(pythonExecutable, pythonScriptPath, string(sqlfluffConfigPath), string(queryString))
5555

@@ -61,19 +61,52 @@ func formatSqlCode(sqlxFileMetaData *sqlxFileMetaData, pythonScriptPath string,
6161
err := cmd.Run()
6262
if err != nil {
6363
logger.Error(stderr.String(), slog.String("file", sqlxFileMetaData.filepath), "error", err.Error())
64-
sqlxFileMetaData.formattedQuery = string(queryString) // If there is an error, return the original query
64+
sqlxFileMetaData.sqlBlocksMeta.formattedSqlBlockContent = string(queryString)
6565
return ErrorFormattingSqlxFile
6666
}
6767
output := stdout.String()
6868
sql_fluff_not_installed := (strings.TrimSpace(output) == "sqlfluff is not installed")
6969
if sql_fluff_not_installed {
7070
log.Fatal(color.RedString("sqlfluff not installed. Please install sqlfluff using 'pip install sqlfluff'"))
7171
}
72-
sqlxFileMetaData.formattedQuery = output
72+
sqlxFileMetaData.sqlBlocksMeta.formattedSqlBlockContent = output
7373
return nil
7474
}
7575

76-
func writeContentsToFile(sqlxFileMetaData *sqlxFileMetaData, formattingError error) {
76+
func finalFormmatedSqlxFileContents(sqlxFileMetaData *sqlxParserMeta) string {
77+
spaceBetweenBlocks := "\n\n"
78+
spaceBetweenSameOps := "\n"
79+
80+
formattedQuery := ""
81+
82+
preOpsBlocks := sqlxFileMetaData.preOpsBlocksMeta
83+
postOpsBlocks := sqlxFileMetaData.postOpsBlocksMeta
84+
85+
preOpsBlockContent := ""
86+
if len(preOpsBlocks) > 0 {
87+
for _, preOpsBlock := range preOpsBlocks {
88+
preOpsBlockContent += preOpsBlock.preOpsBlockContent + spaceBetweenSameOps
89+
}
90+
}
91+
92+
postOpsBlockContent := ""
93+
if len(postOpsBlocks) > 0 {
94+
for _, postOpsBlock := range postOpsBlocks {
95+
postOpsBlockContent += postOpsBlock.postOpsBlockContent + spaceBetweenSameOps
96+
}
97+
}
98+
99+
formattedQuery = sqlxFileMetaData.configBlockMeta.configBlockContent +
100+
spaceBetweenBlocks +
101+
preOpsBlockContent +
102+
spaceBetweenBlocks +
103+
postOpsBlockContent +
104+
spaceBetweenBlocks +
105+
sqlxFileMetaData.sqlBlocksMeta.formattedSqlBlockContent
106+
return formattedQuery
107+
}
108+
109+
func writeContentsToFile(sqlxFileMetaData *sqlxParserMeta, formattingError error) {
77110

78111
yellow := color.New(color.FgYellow).SprintFunc()
79112
red := color.New(color.FgRed).SprintFunc()
@@ -87,14 +120,9 @@ func writeContentsToFile(sqlxFileMetaData *sqlxFileMetaData, formattingError err
87120

88121
os.MkdirAll(dirToCreate, 0755) // TODO: make this configurable
89122

90-
completeQuery := ""
91-
if sqlxFileMetaData.preOperationsString == "" {
92-
completeQuery = sqlxFileMetaData.configString + "\n\n" + sqlxFileMetaData.formattedQuery
93-
} else {
94-
completeQuery = sqlxFileMetaData.configString + "\n\n" + sqlxFileMetaData.preOperationsString + "\n\n" + sqlxFileMetaData.formattedQuery
95-
}
123+
formattedQuery := finalFormmatedSqlxFileContents(sqlxFileMetaData)
96124

97-
err := os.WriteFile(formattedFilePath, []byte(completeQuery), 0664)
125+
err := os.WriteFile(formattedFilePath, []byte(formattedQuery), 0664)
98126
if err != nil {
99127
fmt.Println("Error writing to file:", err)
100128
return
@@ -108,18 +136,14 @@ func writeContentsToFile(sqlxFileMetaData *sqlxFileMetaData, formattingError err
108136
}
109137
}
110138

111-
func writeContentsToFileInPlace(sqlxFileMetaData *sqlxFileMetaData, formattingError error) {
139+
func writeContentsToFileInPlace(sqlxFileMetaData *sqlxParserMeta, formattingError error) {
112140

113141
yellow := color.New(color.FgYellow).SprintFunc()
114142
red := color.New(color.FgRed).SprintFunc()
115143

116-
completeQuery := ""
117-
if sqlxFileMetaData.preOperationsString == "" {
118-
completeQuery = sqlxFileMetaData.configString + "\n\n" + sqlxFileMetaData.formattedQuery
119-
} else {
120-
completeQuery = sqlxFileMetaData.configString + "\n\n" + sqlxFileMetaData.preOperationsString + "\n\n" + sqlxFileMetaData.formattedQuery
121-
}
122-
err := os.WriteFile(sqlxFileMetaData.filepath, []byte(completeQuery), 0664)
144+
formattedQuery := finalFormmatedSqlxFileContents(sqlxFileMetaData)
145+
146+
err := os.WriteFile(sqlxFileMetaData.filepath, []byte(formattedQuery), 0664)
123147
if err != nil {
124148
fmt.Println("Error writing to file:", err)
125149
return
@@ -134,8 +158,7 @@ func writeContentsToFileInPlace(sqlxFileMetaData *sqlxFileMetaData, formattingEr
134158
}
135159

136160
func formatSqlxFile(sqlxFilePath string, inplace bool, sqlfluffConfigPath string, pythonExecutable string, logger *slog.Logger) {
137-
sqlxFileMetaData, err := getSqlxFileMetaData(sqlxFilePath)
138-
161+
sqlxFileMetaData, err := sqlxParser(sqlxFilePath)
139162
if err != nil {
140163
fmt.Println("Error finding config blocks:", err)
141164
} else {

0 commit comments

Comments
 (0)