Merge pull request #305 from breml/generate-data-sources

Generate data sources for network and storage (and more)

Author: stgraber

.gitattributes

@@ -0,0 +1,16 @@
+# Auto detect text files and perform LF normalization
+*                   text=auto
+
+# Collapse generated and vendored files on GitHub
+*_gen.*             linguist-generated merge=ours
+*_gen_test.*        linguist-generated merge=ours
+go.sum              linguist-generated merge=ours
+go.mod              linguist-generated
+
+# Reduce conflicts on markdown files
+*.md                merge=union
+
+# A set of files you probably don't want in distribution
+/.github            export-ignore
+.gitattributes      export-ignore
+.gitignore          export-ignore

CONTRIBUTING.md

@@ -45,9 +45,10 @@ Separate commits should be used for:
 
 - Documentation (`docs: Update XYZ` for files in `docs/`)
 - Resources (`<resource>: Add XYZ` for changes to resources in `internal/`)
+- Generated (`<resource>: Generated XYZ` for changes to resources in `internal/`)
 - Tests (`tests: Add test for XYZ` for changes to `tests/`)
 
-This structure makes it easiser for contributions to be reviewed.
+This structure makes it easier for contributions to be reviewed.
 
 ### Developer Certificate of Origin
 

Makefile

@@ -15,6 +15,9 @@ testacc:
 build:
 	$(GO) build -v
 
+generate:
+	$(GO) generate ./...
+
 targets:
 	gox -osarch='$(TARGETS)' -output="dist/{{.OS}}_{{.Arch}}/terraform-provider-incus_${TRAVIS_TAG}_x4"
 	find dist -maxdepth 1 -mindepth 1 -type d -print0 | \

README.md

@@ -52,6 +52,29 @@ provider_installation {
 }
 ```
 
+#### Generated code
+
+Most of the data sources are generated using the tool located in
+`./cmd/generate-datasources`. The generated files have the suffix `_gen.go` as
+well as a comment `// Code generated by generate-datasources; DO NOT EDIT.`.
+Do not edit these files manually but instead change the respective template in
+`./cmd/generate-datasources/tmpl` and the generator if necessary.
+
+The generator tool is controlled by the config file `generate-datasources.yaml`.
+
+If the generator code, the templates of the generator's config file are changed,
+the generated files need to be regenerated.
+
+Use the following command to regenerate the data sources:
+
+```shell
+make generate
+```
+
+For more details about the inner working of the generator tool as well as the
+settings in the config file please refer to
+[`cmd/generate-datasources/README.md`](cmd/generate-datasources/README.md).
+
 #### Testing
 
 There are two test suites, unit and acceptance. By default the acceptance tests are not run as they require a functional

cmd/generate-datasources/README.md

@@ -0,0 +1,72 @@
+# generate-datasources
+
+`generate-datasources` is a tool to generate data sources for the the Terraform
+provider. It is controlled by the config file `generate-datasources.yaml`.
+
+## Usage
+
+```shell
+go run ./cmd/generate-datasources
+```
+
+For development or debugging of `generate-datasources`, the following flags might be helpful:
+
+```none
+Usage of generate-datasources:
+      --config string          filename of the configfile (default "generate-datasources.yaml")
+  -d, --debug                  Show all debug messages
+      --only-entity string     Limit code generation to this entity
+      --only-template string   Limit code generation to this template
+  -v, --verbose                Show all information messages
+```
+
+With `--only-*`, the generator can be instructed to only generate some defined parts:
+
+* `entity` is the name of the entity as defined in the config file, e.g. `network`.
+* `template` is the name of the template file in `./cmd/generate-datasources/tmpl`, e.g. `datasource.go.gotmpl`.
+
+## Config `generate-datasources.yaml`
+
+The config settings available per entity, for which the data source should be
+generated, are defined and documented in [config.go](./config.go).
+
+## Templates
+
+The templates used by `generate-datasources` are [Go templates](https://pkg.go.dev/text/template).
+
+Templating is used for two purposes:
+
+* target file name and path
+* content of the generated file
+
+There are two kinds of template files:
+
+* regular: these templates are executed for each entity (resource).
+* global: these templates are only executed once for all entities.
+
+### Arguments
+
+For the *regular* templates, the arguments defined in `entityArgs` for the
+currently generated entity are available.
+
+For *global* templates, a map with all entities is passed, where the key is
+the name of the entity (as provided in the config file) and the value is the
+respective `entityArgs` instance.
+
+The type `entityArgs` is defined and documented in [main.go](./main.go).
+
+### Functions
+
+Additionally to the functions and operators provided by  [Go templates](https://pkg.go.dev/text/template) also the complete set of functions provided by the [sprig](https://masterminds.github.io/sprig/) library is available, with some functions being replaced with equivalents
+(see below).
+
+The following specialized functions are provided:
+
+* `pascalcase`: Convert a string in snake case to pascal case (`some_name` -> `SomeName`)
+* `camelcase`: Convert a string in snake case to camel case (`some_name` -> `someName`)
+* `kebabcase`: Convert a string in snake case to kebab case (`some_name` -> `some-name`)
+* `titlecase`: Convert a string in snake case to title case (`some_name` -> `Some Name`)
+* `words`: Split a string in snake case into words (`some_name` -> `some name`)
+
+These functions do value acronyms and will convert them to the appropriate case as well.
+For the list of supported acronyms, please refer to [funcs.go](./funcs.go).

cmd/generate-datasources/config.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Config contains a map of the specification for all entities (resources),
+// for which the code for the data source should be generated.
+// The key of the map is the `name` of the resource, e.g. `network`.
+// The key MUST be singular and in snake case, e.g. `network_forward`.
+type Config map[string]*Entity
+
+type Entity struct {
+	// Additional description about the resource. This is added to the
+	// introduction section of resource in the documentation.
+	Description string `yaml:"description"`
+
+	// Content for the optinal notes section at the end of the data source
+	// documention.
+	Notes string `yaml:"notes"`
+
+	// Name of the package, this data source belongs to.
+	// This also defines the path of the package in `internal/`.
+	PackageName string `yaml:"package-name"`
+
+	// Name property of the resource. If not set, this defaults to `name`.
+	// But some entities do not have a `name` property and therefore an other
+	// property is name defining.
+	ObjectNamePropertyName string `yaml:"object-name-property-name"`
+
+	// Default value used in the documentation for the name defining property.
+	// Defaults to: `default`.
+	ObjectNamePropertyDefaultValue string `yaml:"object-name-property-default-value"`
+
+	// Method of the Incus client to get the resource, e.g. `GetNetwork`.
+	//
+	// E.g. from https://github.com/lxc/incus/blob/3da8fcd06c4f7ee3cb9388127e6071244db7ac8f/client/incus_networks.go#L104
+	IncusGetMethod string `yaml:"incus-get-method"`
+
+	// Name of the parent entity, if any.
+	// Resources like network forwards have a parent, in this case a network.
+	// If this is the case, the name of the parent needs to be specidied.
+	ParentName string `yaml:"parent"`
+
+	// If a resource has no project attribute.
+	// Most resources do have a project attribute. If this is not the case,
+	// `has-no-project` needs to be set to `true`.
+	HasNoProject bool `yaml:"has-no-project"`
+
+	// If a resource has no status attribute.
+	// Most resources do have a status attribute. If this is not the case,
+	// `has-no-status` needs to be set to `true`.
+	HasNoStatus bool `yaml:"has-no-status"`
+
+	// If a resource has a location, mutual exclusive with has-locations.
+	// Some resources are location aware. If this is the case for a resource,
+	// `has-location` needs to be set to `true`.
+	HasLocation bool `yaml:"has-location"`
+
+	// If a resource has multiple locations, mutual exclusive with has-location.
+	// Some resources can be assigned to multiple locations. If this is the case
+	// for a resource, `has-locations` needs to be set to `true`.
+	HasLocations bool `yaml:"has-locations"`
+
+	// Name in snake case of an extra ID attribute, which is specific to the
+	// respective resource type.
+	// As of now, the only resource requiring an extra ID defining attribute is
+	// storage volume, where the `type` is also part of the ID.
+	ExtraIDAttribute ExtraAttribute `yaml:"extra-id-attribute"`
+
+	// List of extra attributes, which are specific to the respective resource type.
+	//
+	// See definition of type ExtraAttribute for details.
+	//
+	// The following attributes are handled automatically be the code generator
+	// and must therefore not be listed in `extra-attributes`:
+	//
+	//   * `name` (or if the resource does not have a `name` attribute, the attribute referenced in `object-name-property-name`)
+	//   * The attributes mentioned in `extra-id-attribute`, if any
+	//   * `parent` (if `parent` is not empty)
+	//   * `project`
+	//   * `target`
+	//   * `remote`
+	//   * `description`
+	//   * `config`
+	//   * `status` (if `has-no-status` is not set to `true`)
+	//   * `location` (if `has-location` is set to `true`)
+	//   * `locations` (if `has-locations` is set to `true`)
+	ExtraAttributes []ExtraAttribute `yaml:"extra-attributes"`
+
+	// Map of additional description added to the documentation for the
+	// automatically handled attributes (see list above).
+	// Is added to the documentation "as-is", may contain markdown.
+	ExtraDescriptions map[string]string `yaml:"extra-descriptions"`
+}
+
+// ExtraAttribute contains the specification for an attribute of a resource.
+type ExtraAttribute struct {
+	// Name of the attribute in snake case, e.g. `type`.
+	Name string `yaml:"name"`
+
+	// Data type of the attribute, e.g. `string`.
+	// Supported types are (mapping directly to the corresponding types from Terraform):
+	//
+	//   * `bool`
+	//   * `list`
+	//   * `map`
+	//   * `object`
+	//   * `string`
+	//
+	// Additionally, the following special types are supported:
+	// (the internal types are recognizable by the leading `_`)
+	//
+	//   * `_device`: represents devices as used in instances and profiles. On the
+	//                API, these are represented as map[string]map[string]string,
+	//                which is mapped to a set of blocks of device information
+	//                consisting of name, type and properties.
+	//                This is a special type with the names and the logic hard
+	//                coded. With this, it can only be used to represent devices.
+	Type string `yaml:"type"`
+
+	// If the `type` is `list` or `map`, `element-type` defines the Terraform type
+	// of the elements contained in the list or map.
+	// Supported types for list are:
+	//
+	//   * `bool`
+	//   * `list`
+	//   * `map`
+	//   * `object`
+	//   * `string`
+	//
+	// Supported types for `map`:
+	//
+	//   * `bool`
+	//   * `list`
+	//   * `object`
+	//   * `string`
+	//
+	// Be aware, that nesting of map in map is not supported by Terraform.
+	ElementType *ExtraAttribute `yaml:"element-type"`
+
+	// If the `type` or the `element-type` is `object`, the type of the attribtues of
+	// the object need to be defined. This is a list, listing each possible attribute of
+	// the object.
+	AttrTypes []*ExtraAttribute `yaml:"attr-types"`
+
+	// Description of the extra attribute. This is added to the documentation.
+	Description string `yaml:"description"`
+}
+
+func (c *Config) LoadConfig(path string) error {
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+
+	return yaml.Unmarshal(contents, c)
+}

cmd/generate-datasources/funcs.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+)
+
+// Capital capitalizes the given string ("foo" -> "Foo").
+func Capital(s string) string {
+	return cases.Title(language.English, cases.NoLower).String(s)
+}
+
+var acronyms = map[string]struct{}{
+	"acl":  {},
+	"url":  {},
+	"snat": {},
+}
+
+// CamelCase converts to camel case ("foo_bar" -> "fooBar").
+// If a segment (with the exception of the first one) is a known acronym,
+// it is returned in all upper case.
+func CamelCase(s string) string {
+	words := capitalizedWords(s)
+	words[0] = strings.ToLower(words[0])
+	return strings.Join(words, "")
+}
+
+// PascalCase converts to pascal case ("foo_bar" -> "FooBar").
+// If a segment is a known acronym, it is returned in all upper case.
+func PascalCase(s string) string {
+	return strings.Join(capitalizedWords(s), "")
+}
+
+// KebabCase converts to kebab case ("foo_bar" -> "foo-bar").
+func KebabCase(s string) string {
+	return strings.ToLower(strings.Join(capitalizedWords(s), "-"))
+}
+
+// TitleCase converts to title case ("foo_bar" -> "Foo Bar").
+// If a segment is a known acronym, it is returned in all upper case.
+func TitleCase(s string) string {
+	return strings.Join(capitalizedWords(s), " ")
+}
+
+// Words converts to space delimited words ("foo_bar" -> "foo bar").
+// If a segment is a known acronym, it is returned in all upper case.
+func Words(s string) string {
+	words := capitalizedWords(s)
+	for i, w := range words {
+		runes := []rune(w)
+		if len(runes) > 1 && unicode.IsUpper(runes[1]) {
+			continue
+		}
+
+		words[i] = strings.ToLower(w)
+	}
+
+	return strings.Join(words, " ")
+}
+
+func capitalizedWords(s string) []string {
+	words := strings.Split(s, "_")
+	for i := range words {
+		_, ok := acronyms[strings.ToLower(words[i])]
+		if ok {
+			words[i] = strings.ToUpper(words[i])
+			continue
+		}
+
+		words[i] = Capital(words[i])
+	}
+
+	return words
+}