Command line code generator for supporting files

README.md

@@ -73,6 +73,49 @@ func main() {
 }
 ```
 
+# Generate support for third-party modules
+This is an example that takes the path of a third-party module source code to generate support for it. Assume the source code path of that module is `./src`. You can call `go run cmd/generate/main.go ./src`. The stdout will be like
+
+```go
+/**
+ * Copyright (c) F5, Inc.
+ *
+ * This source code is licensed under the Apache License, Version 2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Code generated by generator; DO NOT EDIT.
+// All the definitions are extracted from the source code
+// Each bit mask describes these behaviors:
+//   - how many arguments the directive can take
+//   - whether or not it is a block directive
+//   - whether this is a flag (takes one argument that's either "on" or "off")
+//   - which contexts it's allowed to be in
+
+package crossplane
+
+var directives = map[string][]uint{
+    "my_directive_1": {
+        bitmask01|bitmask02|...,
+		bitmask11|bitmask12|...,
+		...
+    },
+    "my_directive_2": {
+        bitmask01|bitmask02|...,
+		bitmask11|bitmask12|...,
+		...
+    },
+}
+
+// Match is a matchFunc for parsing an NGINX config that contains the
+// preceding directives.
+func Match(directive string) ([]uint, bool) {
+    m, ok := directives[directive]
+    return m, ok
+}
+```
+You can redirect the stdout into a `.go` file, and pass the generated `matchFunc` to `ParseOptions.DirectiveSources` when invoking `Parse`.
+
 ## Contributing
 
 If you'd like to contribute to the project, please read our [Contributing guide](CONTRIBUTING.md).

cmd/generate/main.go

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) F5, Inc.
+ *
+ * This source code is licensed under the Apache License, Version 2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+
+	"github.com/nginxinc/nginx-go-crossplane/internal/generator"
+)
+
+func main() {
+	var (
+		sourceCodePath = flag.String("src-path", "",
+			"the path of source code your want to generate support from, it can be either a file or a directory. (required)")
+	)
+	flag.Parse()
+	err := generator.Generate(*sourceCodePath, os.Stdout)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -27,11 +27,12 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
 golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/generator/generator.go

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) F5, Inc.
+ *
+ * This source code is licensed under the Apache License, Version 2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package generator
+
+import (
+	"io"
+)
+
+// Generate receives a string sourcePath and an io.Writer writer. It will
+// extract all the directives definitions from the .c and .cpp files in
+// sourcePath and its subdirectories, then output the corresponding directive
+// masks map named "directives" and matchFunc named "Match" via writer.
+func Generate(sourcePath string, writer io.Writer) error {
+	return genFromSrcCode(sourcePath, "directives", "Match", writer)
+}

internal/generator/generator_util.go

@@ -0,0 +1,203 @@
+/**
+ * Copyright (c) F5, Inc.
+ *
+ * This source code is licensed under the Apache License, Version 2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package generator
+
+import (
+	_ "embed"
+	"errors"
+	"fmt"
+	"html/template"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// A mask is a list of string, includes several variable names,
+// which specify a behavior of a directive.
+// An example is []string{"ngxHTTPMainConf", "ngxConfFlag",}.
+// A directive can have several masks.
+type mask []string
+
+type supportFileTmplStruct struct {
+	Directive2Masks map[string][]mask
+	MapVariableName string
+	MatchFnName     string
+}
+
+var (
+	// Extract single directive definition block
+	// static ngx_command_t  {name}[] = {definition}
+	// this regex extracts {name} and {definition}.
+	directivesDefineBlockExtracter = regexp.MustCompile(`ngx_command_t\s+(\w+)\[\]\s*=\s*{(.*?)}\s*;`)
+
+	// Extract one directive definition and attributes from extracted block
+	// { ngx_string({directive_name}),
+	//   {bitmask1|bitmask2|...},
+	//   ... },
+	// this regex extracts {directive_name} and {bitmask1|bitmask2|...}.
+	singleDirectiveExtracter = regexp.MustCompile(`ngx_string\("(.*?)"\).*?,(.*?),`)
+
+	singleLineCommentExtracter = regexp.MustCompile(`//.*`)
+
+	multiLineCommentExtracter = regexp.MustCompile(`/\*[\s\S]*?\*/`)
+)
+
+// Template of support file. A support file contains a map from
+// diective to its bitmask definitions, and a MatchFunc for it.
+//
+//go:embed tmpl/support_file.tmpl
+var supportFileTmplStr string
+
+//nolint:gochecknoglobals
+var supportFileTmpl = template.Must(template.New("supportFile").
+	Funcs(template.FuncMap{"Join": strings.Join}).Parse(supportFileTmplStr))
+
+//nolint:gochecknoglobals
+var ngxVarNameToGo = map[string]string{
+	"NGX_MAIL_MAIN_CONF":   "ngxMailMainConf",
+	"NGX_STREAM_MAIN_CONF": "ngxStreamMainConf",
+	"NGX_CONF_TAKE1":       "ngxConfTake1",
+	"NGX_STREAM_UPS_CONF":  "ngxStreamUpsConf",
+	"NGX_HTTP_LIF_CONF":    "ngxHTTPLifConf",
+	"NGX_CONF_TAKE2":       "ngxConfTake2",
+	"NGX_HTTP_UPS_CONF":    "ngxHTTPUpsConf",
+	"NGX_CONF_TAKE23":      "ngxConfTake23",
+	"NGX_CONF_TAKE12":      "ngxConfTake12",
+	"NGX_HTTP_MAIN_CONF":   "ngxHTTPMainConf",
+	"NGX_HTTP_LMT_CONF":    "ngxHTTPLmtConf",
+	"NGX_CONF_TAKE1234":    "ngxConfTake1234",
+	"NGX_MAIL_SRV_CONF":    "ngxMailSrvConf",
+	"NGX_CONF_FLAG":        "ngxConfFlag",
+	"NGX_HTTP_SRV_CONF":    "ngxHTTPSrvConf",
+	"NGX_CONF_1MORE":       "ngxConf1More",
+	"NGX_ANY_CONF":         "ngxAnyConf",
+	"NGX_CONF_TAKE123":     "ngxConfTake123",
+	"NGX_MAIN_CONF":        "ngxMainConf",
+	"NGX_CONF_NOARGS":      "ngxConfNoArgs",
+	"NGX_CONF_2MORE":       "ngxConf2More",
+	"NGX_CONF_TAKE3":       "ngxConfTake3",
+	"NGX_HTTP_SIF_CONF":    "ngxHTTPSifConf",
+	"NGX_EVENT_CONF":       "ngxEventConf",
+	"NGX_CONF_BLOCK":       "ngxConfBlock",
+	"NGX_HTTP_LOC_CONF":    "ngxHTTPLocConf",
+	"NGX_STREAM_SRV_CONF":  "ngxStreamSrvConf",
+	"NGX_DIRECT_CONF":      "ngxDirectConf",
+	"NGX_CONF_TAKE13":      "ngxConfTake13",
+	"NGX_CONF_ANY":         "ngxConfAny",
+	"NGX_CONF_TAKE4":       "ngxConfTake4",
+	"NGX_CONF_TAKE5":       "ngxConfTake5",
+	"NGX_CONF_TAKE6":       "ngxConfTake6",
+	"NGX_CONF_TAKE7":       "ngxConfTake7",
+}
+
+//nolint:nonamedreturns
+func masksFromFile(path string) (directive2Masks map[string][]mask, err error) {
+	directive2Masks = make(map[string][]mask, 0)
+	byteContent, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	strContent := string(byteContent)
+
+	// Remove comments
+	strContent = singleLineCommentExtracter.ReplaceAllString(strContent, "")
+	strContent = multiLineCommentExtracter.ReplaceAllString(strContent, "")
+	strContent = strings.ReplaceAll(strContent, "\r\n", "")
+	strContent = strings.ReplaceAll(strContent, "\n", "")
+
+	// Extract directives definition code blocks, each code block contains a list of directives definition
+	blocks := directivesDefineBlockExtracter.FindAllStringSubmatch(strContent, -1)
+
+	for _, block := range blocks {
+		// Extract directives and their attributes in the code block, the first dimension of subBlocks
+		// is index of directive, the second dimension is index of attributes
+		subBlocks := singleDirectiveExtracter.FindAllStringSubmatch(block[2], -1)
+
+		// Iterate through every directive
+		for _, attributes := range subBlocks {
+			// Extract attributes from the directive
+			directiveName := strings.TrimSpace(attributes[1])
+			directiveMask := strings.Split(attributes[2], "|")
+
+			// Transfer C-style mask to go style
+			for idx, ngxVarName := range directiveMask {
+				goVarName, found := ngxVarNameToGo[strings.TrimSpace(ngxVarName)]
+				if !found {
+					return nil, fmt.Errorf("parsing directive %s, bitmask %s in source code not found in crossplane", directiveName, ngxVarName)
+				}
+				directiveMask[idx] = goVarName
+			}
+
+			directive2Masks[directiveName] = append(directive2Masks[directiveName], directiveMask)
+		}
+	}
+	return directive2Masks, nil
+}
+
+//nolint:nonamedreturns
+func getMasksFromPath(path string) (directive2Masks map[string][]mask, err error) {
+	directive2Masks = make(map[string][]mask, 0)
+
+	err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Check if the entry is a C/C++ file
+		// Some dynamic modules are written in C++, like otel
+		if d.IsDir() {
+			return nil
+		}
+
+		if !(strings.HasSuffix(path, ".c") || strings.HasSuffix(path, ".cpp")) {
+			return nil
+		}
+
+		directive2MasksInFile, err := masksFromFile(path)
+		if err != nil {
+			return err
+		}
+
+		for directive, masksInFile := range directive2MasksInFile {
+			directive2Masks[directive] = append(directive2Masks[directive], masksInFile...)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	if len(directive2Masks) == 0 {
+		return nil, errors.New("can't find any directives in the directory and subdirectories, please check the path")
+	}
+
+	return directive2Masks, nil
+}
+
+func genFromSrcCode(codePath string, mapVariableName string, matchFnName string, writer io.Writer) error {
+	directive2Masks, err := getMasksFromPath(codePath)
+	if err != nil {
+		return err
+	}
+
+	err = supportFileTmpl.Execute(writer, supportFileTmplStruct{
+		Directive2Masks: directive2Masks,
+		MapVariableName: mapVariableName,
+		MatchFnName:     matchFnName,
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}