From 8781f2f14a544399835b87a0845c9cbeb00ef459 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 12 Feb 2024 01:01:42 -0800 Subject: [PATCH 01/71] add query type support --- experimental/query/common.go | 70 ++++++++++ experimental/query/common.jsonschema | 78 +++++++++++ experimental/query/common_test.go | 41 ++++++ experimental/query/definition.go | 64 +++++++++ experimental/query/expr/math.go | 43 +++++++ experimental/query/expr/reduce.go | 57 ++++++++ experimental/query/expr/resample.go | 24 ++++ experimental/query/expr/types.go | 66 ++++++++++ experimental/query/expr/types_test.go | 42 ++++++ experimental/query/schema/builder.go | 164 ++++++++++++++++++++++++ experimental/query/schema/enums.go | 115 +++++++++++++++++ experimental/query/schema/enums_test.go | 20 +++ go.mod | 4 + go.sum | 8 ++ 14 files changed, 796 insertions(+) create mode 100644 experimental/query/common.go create mode 100644 experimental/query/common.jsonschema create mode 100644 experimental/query/common_test.go create mode 100644 experimental/query/definition.go create mode 100644 experimental/query/expr/math.go create mode 100644 experimental/query/expr/reduce.go create mode 100644 experimental/query/expr/resample.go create mode 100644 experimental/query/expr/types.go create mode 100644 experimental/query/expr/types_test.go create mode 100644 experimental/query/schema/builder.go create mode 100644 experimental/query/schema/enums.go create mode 100644 experimental/query/schema/enums_test.go diff --git a/experimental/query/common.go b/experimental/query/common.go new file mode 100644 index 000000000..f1720f4c9 --- /dev/null +++ b/experimental/query/common.go @@ -0,0 +1,70 @@ +package query + +import "embed" + +type CommonQueryProperties struct { + // RefID is the unique identifier of the query, set by the frontend call. + RefID string `json:"refId,omitempty"` + + // TimeRange represents the query range + // NOTE: unlike generic /ds/query, we can now send explicit time values in each query + TimeRange *TimeRange `json:"timeRange,omitempty"` + + // The datasource + Datasource *DataSourceRef `json:"datasource,omitempty"` + + // Deprecated -- use datasource ref instead + DatasourceId int64 `json:"datasourceId,omitempty"` + + // QueryType is an optional identifier for the type of query. + // It can be used to distinguish different types of queries. + QueryType string `json:"queryType,omitempty"` + + // MaxDataPoints is the maximum number of data points that should be returned from a time series query. + MaxDataPoints int64 `json:"maxDataPoints,omitempty"` + + // Interval is the suggested duration between time points in a time series query. + IntervalMS float64 `json:"intervalMs,omitempty"` + + // true if query is disabled (ie should not be returned to the dashboard) + // Note this does not always imply that the query should not be executed since + // the results from a hidden query may be used as the input to other queries (SSE etc) + Hide bool `json:"hide,omitempty"` +} + +type DataSourceRef struct { + // The datasource plugin type + Type string `json:"type"` + + // Datasource UID + UID string `json:"uid"` + + // ?? the datasource API version + // ApiVersion string `json:"apiVersion"` +} + +// TimeRange represents a time range for a query and is a property of DataQuery. +type TimeRange struct { + // From is the start time of the query. + From string `json:"from"` + + // To is the end time of the query. + To string `json:"to"` +} + +// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing +type GenericDataQuery struct { + CommonQueryProperties `json:",inline"` + + // Additional Properties (that live at the root) + Additional map[string]any `json:",inline"` +} + +//go:embed common.jsonschema +var f embed.FS + +// Get the cached feature list (exposed as a k8s resource) +func GetCommonJSONSchema() []byte { + body, _ := f.ReadFile("common.jsonschema") + return body +} diff --git a/experimental/query/common.jsonschema b/experimental/query/common.jsonschema new file mode 100644 index 000000000..dd7c766d3 --- /dev/null +++ b/experimental/query/common.jsonschema @@ -0,0 +1,78 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/common-query-properties", + "$ref": "#/$defs/CommonQueryProperties", + "$defs": { + "CommonQueryProperties": { + "properties": { + "refId": { + "type": "string", + "description": "RefID is the unique identifier of the query, set by the frontend call." + }, + "timeRange": { + "$ref": "#/$defs/TimeRange", + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query" + }, + "datasource": { + "$ref": "#/$defs/DataSourceRef", + "description": "The datasource" + }, + "queryType": { + "type": "string", + "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." + }, + "maxDataPoints": { + "type": "integer", + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query." + }, + "intervalMs": { + "type": "number", + "description": "Interval is the suggested duration between time points in a time series query." + }, + "hide": { + "type": "boolean", + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNote this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" + } + }, + "additionalProperties": false, + "type": "object" + }, + "DataSourceRef": { + "properties": { + "type": { + "type": "string", + "description": "The datasource plugin type" + }, + "uid": { + "type": "string", + "description": "Datasource UID" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type", + "uid" + ] + }, + "TimeRange": { + "properties": { + "from": { + "type": "string", + "description": "From is the start time of the query." + }, + "to": { + "type": "string", + "description": "To is the end time of the query." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "from", + "to" + ], + "description": "TimeRange represents a time range for a query and is a property of DataQuery." + } + } +} \ No newline at end of file diff --git a/experimental/query/common_test.go b/experimental/query/common_test.go new file mode 100644 index 000000000..8932de46d --- /dev/null +++ b/experimental/query/common_test.go @@ -0,0 +1,41 @@ +package query + +import ( + "encoding/json" + "os" + "testing" + + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCommonSupport(t *testing.T) { + r := new(jsonschema.Reflector) + err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/query", "./") + require.NoError(t, err) + + query := r.Reflect(&CommonQueryProperties{}) + common, ok := query.Definitions["CommonQueryProperties"] + require.True(t, ok) + + // Hide this old property + common.Properties.Delete("datasourceId") + out, err := json.MarshalIndent(query, "", " ") + require.NoError(t, err) + + update := false + outfile := "common.jsonschema" + body, err := os.ReadFile(outfile) + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true + } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0644) + require.NoError(t, err, "error writing file") + } +} diff --git a/experimental/query/definition.go b/experimental/query/definition.go new file mode 100644 index 000000000..dd88451ce --- /dev/null +++ b/experimental/query/definition.go @@ -0,0 +1,64 @@ +package query + +import ( + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" +) + +type TypedQueryHandler[Q any] interface { + // QueryTypeField is typically "queryType", but may use a different field to + // discriminate different field types. When multiple versions for a field exist, + // The version identifier is appended to the queryType value after a slash. + // eg: queryType=showLabels/v2 + QueryTypeField() string + + // Possible query types (if any) + QueryTypes() []QueryTypeDefinition + + // Get the query parser for a query type + // The version is split from the end of the discriminator field + ReadQuery( + // The query type split by version (when multiple exist) + queryType string, version string, + // Properties that have been parsed off the same node + common CommonQueryProperties, + // An iterator with context for the full node (include common values) + iter *jsoniter.Iterator, + ) (Q, error) +} + +type QueryTypeDefinitions struct { + // Describe whe the query type is for + Field string `json:"field,omitempty"` + + Types []QueryTypeDefinition `json:"types"` +} + +type QueryTypeDefinition struct { + // Describe whe the query type is for + Name string `json:"name,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // Versions (most recent first) + Versions []QueryTypeVersion `json:"versions"` + + // When multiple versions exist, this is the preferredVersion + PreferredVersion string `json:"preferredVersion,omitempty"` +} + +type QueryTypeVersion struct { + // Version identifier or empty if only one exists + Version string `json:"version,omitempty"` + + // The JSONSchema definition for the non-common fields + Schema any `json:"schema"` + + // Examples (include a wrapper) ideally a template! + Examples []any `json:"examples,omitempty"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} diff --git a/experimental/query/expr/math.go b/experimental/query/expr/math.go new file mode 100644 index 000000000..9e7d2c69c --- /dev/null +++ b/experimental/query/expr/math.go @@ -0,0 +1,43 @@ +package expr + +import "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + +var _ ExpressionQuery = (*MathQuery)(nil) + +type MathQuery struct { + // General math expression + Expression string `json:"expression"` + + // Parsed from the expression + variables []string `json:"-"` +} + +func (*MathQuery) ExpressionQueryType() QueryType { + return QueryTypeMath +} + +func (q *MathQuery) Variables() []string { + return q.variables +} + +func readMathQuery(version string, iter *jsoniter.Iterator) (q *MathQuery, err error) { + fname := "" + for fname, err = iter.ReadObject(); fname != "" && err == nil; fname, err = iter.ReadObject() { + switch fname { + case "expression": + temp, err := iter.ReadString() + if err != nil { + return q, err + } + // TODO actually parse the expression + q.Expression = temp + + default: + _, err = iter.ReadAny() // eat up the unused fields + if err != nil { + return nil, err + } + } + } + return +} diff --git a/experimental/query/expr/reduce.go b/experimental/query/expr/reduce.go new file mode 100644 index 000000000..d58262168 --- /dev/null +++ b/experimental/query/expr/reduce.go @@ -0,0 +1,57 @@ +package expr + +var _ ExpressionQuery = (*ReduceQuery)(nil) + +type ReduceQuery struct { + // Reference to other query results + Expression string `json:"expression"` + + // The reducer + Reducer ReducerID `json:"reducer"` + + // Reducer Options + Settings ReduceSettings `json:"settings"` +} + +func (*ReduceQuery) ExpressionQueryType() QueryType { + return QueryTypeReduce +} + +func (q *ReduceQuery) Variables() []string { + return []string{q.Expression} +} + +type ReduceSettings struct { + // Non-number reduce behavior + Mode ReduceMode `json:"mode"` + + // Only valid when mode is replace + ReplaceWithValue *float64 `json:"replaceWithValue,omitempty"` +} + +// The reducer function +// +enum +type ReducerID string + +const ( + // The sum + ReducerSum ReducerID = "sum" + // The mean + ReducerMean ReducerID = "mean" + ReducerMin ReducerID = "min" + ReducerMax ReducerID = "max" + ReducerCount ReducerID = "count" + ReducerLast ReducerID = "last" +) + +// Non-Number behavior mode +// +enum +type ReduceMode string + +const ( + // Drop non-numbers + ReduceModeDrop ReduceMode = "dropNN" + + // Replace non-numbers + ReduceModeReplace ReduceMode = "replaceNN" +) diff --git a/experimental/query/expr/resample.go b/experimental/query/expr/resample.go new file mode 100644 index 000000000..d28609134 --- /dev/null +++ b/experimental/query/expr/resample.go @@ -0,0 +1,24 @@ +package expr + +// QueryType = resample +type ResampleQuery struct { + // The math expression + Expression string `json:"expression"` + + // The math expression + Window string `json:"window"` + + // The reducer + Downsampler string `json:"downsampler"` + + // The reducer + Upsampler string `json:"upsampler"` +} + +func (*ResampleQuery) ExpressionQueryType() QueryType { + return QueryTypeReduce +} + +func (q *ResampleQuery) Variables() []string { + return []string{q.Expression} +} diff --git a/experimental/query/expr/types.go b/experimental/query/expr/types.go new file mode 100644 index 000000000..22cfbc2ff --- /dev/null +++ b/experimental/query/expr/types.go @@ -0,0 +1,66 @@ +package expr + +import ( + "fmt" + + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + "github.com/grafana/grafana-plugin-sdk-go/experimental/query" +) + +// Supported expression types +// +enum +type QueryType string + +const ( + // Math query type + QueryTypeMath QueryType = "math" + + // Reduce query type + QueryTypeReduce QueryType = "reduce" + + // Reduce query type + QueryTypeResample QueryType = "resample" +) + +type ExpressionQuery interface { + ExpressionQueryType() QueryType + Variables() []string +} + +var _ query.TypedQueryHandler[ExpressionQuery] = (*QueyHandler)(nil) + +type QueyHandler struct{} + +func (*QueyHandler) QueryTypeField() string { + return "queryType" +} + +// QueryTypes implements query.TypedQueryHandler. +func (*QueyHandler) QueryTypes() []query.QueryTypeDefinition { + return []query.QueryTypeDefinition{} +} + +// ReadQuery implements query.TypedQueryHandler. +func (*QueyHandler) ReadQuery( + // The query type split by version (when multiple exist) + queryType string, version string, + // Properties that have been parsed off the same node + common query.CommonQueryProperties, + // An iterator with context for the full node (include common values) + iter *jsoniter.Iterator, +) (ExpressionQuery, error) { + qt := QueryType(queryType) + switch qt { + case QueryTypeMath: + return readMathQuery(version, iter) + + case QueryTypeReduce: + q := &ReduceQuery{} + err := iter.ReadVal(q) + return q, err + + case QueryTypeResample: + return nil, nil + } + return nil, fmt.Errorf("unknown query type") +} diff --git a/experimental/query/expr/types_test.go b/experimental/query/expr/types_test.go new file mode 100644 index 000000000..d9f8f6110 --- /dev/null +++ b/experimental/query/expr/types_test.go @@ -0,0 +1,42 @@ +package expr + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/query/schema" + "github.com/stretchr/testify/require" +) + +func TestQueryTypeDefinitions(t *testing.T) { + builder, err := schema.NewBuilder( + schema.BuilderOptions{ + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", + CodePath: "./", + PluginIDs: []string{"expr"}, + }, + schema.QueryTypeInfo{ + QueryType: string(QueryTypeMath), + GoType: reflect.TypeOf(&MathQuery{}), + }, + schema.QueryTypeInfo{ + QueryType: string(QueryTypeReduce), + GoType: reflect.TypeOf(&ReduceQuery{}), + }, + schema.QueryTypeInfo{ + QueryType: string(QueryTypeResample), + GoType: reflect.TypeOf(&ResampleQuery{}), + }) + require.NoError(t, err) + + // The full schema + s, err := builder.GetFullQuerySchema() + require.NoError(t, err) + + out, err := json.MarshalIndent(s, "", " ") + require.NoError(t, err) + + fmt.Printf("%s\n", out) +} diff --git a/experimental/query/schema/builder.go b/experimental/query/schema/builder.go new file mode 100644 index 000000000..bfec66ad1 --- /dev/null +++ b/experimental/query/schema/builder.go @@ -0,0 +1,164 @@ +package schema + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/query" + "github.com/invopop/jsonschema" +) + +type QueryTypeInfo struct { + QueryType string + Version string + GoType reflect.Type +} + +type QueryTypeBuilder struct { + opts BuilderOptions + reflector *jsonschema.Reflector // Needed to use comments + byType map[string]*query.QueryTypeDefinition + types []*query.QueryTypeDefinition +} + +func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { + schema := b.reflector.ReflectFromType(info.GoType) + if schema == nil { + return fmt.Errorf("missing schema") + } + def, ok := b.byType[info.QueryType] + if !ok { + def = &query.QueryTypeDefinition{ + Name: info.QueryType, + Versions: []query.QueryTypeVersion{}, + } + b.byType[info.QueryType] = def + b.types = append(b.types, def) + } + def.Versions = append(def.Versions, query.QueryTypeVersion{ + Version: info.Version, + Schema: schema, + }) + return nil +} + +type BuilderOptions struct { + // ex "github.com/invopop/jsonschema" + BasePackage string + + // ex "./" + CodePath string + + // queryType + DiscriminatorField string + + // org-xyz-datasource + PluginIDs []string +} + +func NewBuilder(opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder, error) { + r := new(jsonschema.Reflector) + if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { + return nil, err + } + b := &QueryTypeBuilder{ + opts: opts, + reflector: r, + byType: make(map[string]*query.QueryTypeDefinition), + } + for _, input := range inputs { + err := b.Add(input) + if err != nil { + return nil, err + } + } + return b, nil +} + +func (b *QueryTypeBuilder) GetFullQuerySchema() (*jsonschema.Schema, error) { + discriminator := b.opts.DiscriminatorField + if discriminator == "" { + discriminator = "queryType" + } + + query, err := asJSONSchema(query.GetCommonJSONSchema()) + if err != nil { + return nil, err + } + query.Ref = "" + common, ok := query.Definitions["CommonQueryProperties"] + if !ok { + return nil, fmt.Errorf("error finding common properties") + } + delete(query.Definitions, "CommonQueryProperties") + + for _, t := range b.types { + for _, v := range t.Versions { + s, err := asJSONSchema(v.Schema) + if err != nil { + return nil, err + } + if s.Ref == "" { + return nil, fmt.Errorf("only ref elements supported right now") + } + + ref := strings.TrimPrefix(s.Ref, "#/$defs/") + body := s + + // Add all types to the + for key, def := range s.Definitions { + if key == ref { + body = def + } else { + query.Definitions[key] = def + } + } + + if body.Properties == nil { + return nil, fmt.Errorf("expected properties on body") + } + + for pair := common.Properties.Oldest(); pair != nil; pair = pair.Next() { + body.Properties.Set(pair.Key, pair.Value) + } + body.Required = append(body.Required, "refId") + + if t.Name != "" { + key := t.Name + if v.Version != "" { + key += "/" + v.Version + } + + p, err := body.Properties.GetAndMoveToFront(discriminator) + if err != nil { + return nil, fmt.Errorf("missing discriminator field: %s", discriminator) + } + p.Const = key + p.Enum = nil + + body.Required = append(body.Required, discriminator) + } + + query.OneOf = append(query.OneOf, body) + } + } + + return query, nil +} + +// Always creates a copy so we can modify it +func asJSONSchema(v any) (*jsonschema.Schema, error) { + var err error + s := &jsonschema.Schema{} + b, ok := v.([]byte) + if !ok { + b, err = json.Marshal(v) + if err != nil { + return nil, err + } + } + err = json.Unmarshal(b, s) + return s, err +} diff --git a/experimental/query/schema/enums.go b/experimental/query/schema/enums.go new file mode 100644 index 000000000..f3f5e1d1b --- /dev/null +++ b/experimental/query/schema/enums.go @@ -0,0 +1,115 @@ +package schema + +import ( + "fmt" + "io/fs" + gopath "path" + "path/filepath" + "strings" + + "go/ast" + "go/doc" + "go/parser" + "go/token" +) + +type EnumValue struct { + Value string + Comment string +} + +type EnumField struct { + Package string + Name string + Comment string + Values []EnumValue +} + +func FindEnumFields(base, path string) ([]EnumField, error) { + fset := token.NewFileSet() + dict := make(map[string][]*ast.Package) + err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + d, err := parser.ParseDir(fset, path, nil, parser.ParseComments) + if err != nil { + return err + } + for _, v := range d { + // paths may have multiple packages, like for tests + k := gopath.Join(base, path) + dict[k] = append(dict[k], v) + } + } + return nil + }) + if err != nil { + return nil, err + } + + fields := make([]EnumField, 0) + field := &EnumField{} + + for pkg, p := range dict { + for _, f := range p { + gtxt := "" + typ := "" + ast.Inspect(f, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.TypeSpec: + typ = x.Name.String() + if !ast.IsExported(typ) { + typ = "" + } else { + txt := x.Doc.Text() + if txt == "" && gtxt != "" { + txt = gtxt + gtxt = "" + } + txt = strings.TrimSpace(doc.Synopsis(txt)) + if strings.HasSuffix(txt, "+enum") { + fields = append(fields, EnumField{ + Package: pkg, + Name: typ, + Comment: strings.TrimSpace(strings.TrimSuffix(txt, "+enum")), + }) + field = &fields[len(fields)-1] + + fmt.Printf("ENUM: %s.%s // %s\n", pkg, typ, txt) + } + } + case *ast.ValueSpec: + txt := x.Doc.Text() + if txt == "" { + txt = x.Comment.Text() + } + if typ == field.Name { + for _, n := range x.Names { + if ast.IsExported(n.String()) { + v, ok := x.Values[0].(*ast.BasicLit) + if ok { + val := strings.TrimPrefix(v.Value, `"`) + val = strings.TrimSuffix(val, `"`) + txt = strings.TrimSpace(txt) + fmt.Printf("%s // %s // %s\n", typ, val, txt) + field.Values = append(field.Values, EnumValue{ + Value: val, + Comment: txt, + }) + } + } + } + } + case *ast.GenDecl: + // remember for the next type + gtxt = x.Doc.Text() + } + return true + }) + } + } + + return fields, nil +} diff --git a/experimental/query/schema/enums_test.go b/experimental/query/schema/enums_test.go new file mode 100644 index 000000000..1d1442e8a --- /dev/null +++ b/experimental/query/schema/enums_test.go @@ -0,0 +1,20 @@ +package schema + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFindEnums(t *testing.T) { + fields, err := FindEnumFields( + "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", + "../expr") + require.NoError(t, err) + + out, err := json.MarshalIndent(fields, "", " ") + require.NoError(t, err) + fmt.Printf("%s", string(out)) +} diff --git a/go.mod b/go.mod index dd2f3a33b..09acd050b 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,9 @@ require ( require ( github.com/BurntSushi/toml v1.3.2 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -71,6 +73,7 @@ require ( github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 // indirect + github.com/invopop/jsonschema v0.12.0 // indirect github.com/invopop/yaml v0.2.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect @@ -92,6 +95,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/unknwon/com v1.0.1 // indirect github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel/metric v1.22.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect diff --git a/go.sum b/go.sum index fab716237..60ee2326c 100644 --- a/go.sum +++ b/go.sum @@ -9,11 +9,15 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/apache/arrow/go/v15 v15.0.0 h1:1zZACWf85oEZY5/kd9dsQS7i+2G5zVQcbKTHgslqHNA= github.com/apache/arrow/go/v15 v15.0.0/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -113,6 +117,8 @@ github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDm github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= @@ -225,6 +231,8 @@ github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3/go.mod h1:1xEUf2abjfP9 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= From 0ec8629d41bc906f6a31182883f42f13340f78dd Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 12 Feb 2024 16:02:38 -0800 Subject: [PATCH 02/71] move values --- experimental/query/common.go | 30 ++++++- experimental/query/common.jsonschema | 38 +++++++++ experimental/query/definition.go | 18 ++-- experimental/query/expr/math.go | 2 +- experimental/query/expr/types.go | 4 +- experimental/query/expr/types.json | 113 ++++++++++++++++++++++++++ experimental/query/expr/types_test.go | 33 ++++++-- experimental/query/schema/builder.go | 28 ++++--- 8 files changed, 235 insertions(+), 31 deletions(-) create mode 100644 experimental/query/expr/types.json diff --git a/experimental/query/common.go b/experimental/query/common.go index f1720f4c9..0d4688814 100644 --- a/experimental/query/common.go +++ b/experimental/query/common.go @@ -1,11 +1,19 @@ package query -import "embed" +import ( + "embed" + "encoding/json" + + "github.com/grafana/grafana-plugin-sdk-go/data" +) type CommonQueryProperties struct { // RefID is the unique identifier of the query, set by the frontend call. RefID string `json:"refId,omitempty"` + // Optionally define expected query result behavior + ResultAssertions *ResultAssertions `json:"resultAssertions,omitempty"` + // TimeRange represents the query range // NOTE: unlike generic /ds/query, we can now send explicit time values in each query TimeRange *TimeRange `json:"timeRange,omitempty"` @@ -52,6 +60,24 @@ type TimeRange struct { To string `json:"to"` } +// ResultAssertions define the expected response shape and query behavior. This is useful to +// enforce behavior over time. The assertions are passed to the query engine and can be used +// to fail queries *before* returning them to a client (select * from bigquery!) +type ResultAssertions struct { + // Type asserts that the frame matches a known type structure. + Type data.FrameType `json:"type,omitempty"` + + // TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane + // contract documentation https://grafana.github.io/dataplane/contract/. + TypeVersion data.FrameTypeVersion `json:"typeVersion"` + + // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast + MaxBytes int64 `json:"maxBytes,omitempty"` + + // Maximum frame count + MaxFrames int64 `json:"maxFrames,omitempty"` +} + // GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing type GenericDataQuery struct { CommonQueryProperties `json:",inline"` @@ -64,7 +90,7 @@ type GenericDataQuery struct { var f embed.FS // Get the cached feature list (exposed as a k8s resource) -func GetCommonJSONSchema() []byte { +func GetCommonJSONSchema() json.RawMessage { body, _ := f.ReadFile("common.jsonschema") return body } diff --git a/experimental/query/common.jsonschema b/experimental/query/common.jsonschema index dd7c766d3..4f3a02d34 100644 --- a/experimental/query/common.jsonschema +++ b/experimental/query/common.jsonschema @@ -9,6 +9,10 @@ "type": "string", "description": "RefID is the unique identifier of the query, set by the frontend call." }, + "resultAssertions": { + "$ref": "#/$defs/ResultAssertions", + "description": "Optionally define expected query result behavior" + }, "timeRange": { "$ref": "#/$defs/TimeRange", "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query" @@ -55,6 +59,40 @@ "uid" ] }, + "FrameTypeVersion": { + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2 + }, + "ResultAssertions": { + "properties": { + "type": { + "type": "string", + "description": "Type asserts that the frame matches a known type structure." + }, + "typeVersion": { + "$ref": "#/$defs/FrameTypeVersion", + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." + }, + "maxBytes": { + "type": "integer", + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" + }, + "maxFrames": { + "type": "integer", + "description": "Maximum frame count" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "typeVersion" + ], + "description": "ResultAssertions define the expected response shape and query behavior." + }, "TimeRange": { "properties": { "from": { diff --git a/experimental/query/definition.go b/experimental/query/definition.go index dd88451ce..25bdcd976 100644 --- a/experimental/query/definition.go +++ b/experimental/query/definition.go @@ -12,7 +12,7 @@ type TypedQueryHandler[Q any] interface { QueryTypeField() string // Possible query types (if any) - QueryTypes() []QueryTypeDefinition + QueryTypeDefinitions() []QueryTypeDefinitionSpec // Get the query parser for a query type // The version is split from the end of the discriminator field @@ -26,16 +26,14 @@ type TypedQueryHandler[Q any] interface { ) (Q, error) } -type QueryTypeDefinitions struct { - // Describe whe the query type is for - Field string `json:"field,omitempty"` - - Types []QueryTypeDefinition `json:"types"` -} +type QueryTypeDefinitionSpec struct { + // The query type value + // NOTE: this must be a k8s compatible name + Name string `json:"name,omitempty"` // must be k8s name? compatible -type QueryTypeDefinition struct { - // Describe whe the query type is for - Name string `json:"name,omitempty"` + // DiscriminatorField is the field used to link behavior to this specific + // query type. It is typically "queryType", but can be another field if necessary + DiscriminatorField string `json:"discriminatorField,omitempty"` // Describe whe the query type is for Description string `json:"description,omitempty"` diff --git a/experimental/query/expr/math.go b/experimental/query/expr/math.go index 9e7d2c69c..ddc42a57c 100644 --- a/experimental/query/expr/math.go +++ b/experimental/query/expr/math.go @@ -6,7 +6,7 @@ var _ ExpressionQuery = (*MathQuery)(nil) type MathQuery struct { // General math expression - Expression string `json:"expression"` + Expression string `json:"expression" jsonschema:"minLength=1,example=$A + 1,example=$A/$B"` // Parsed from the expression variables []string `json:"-"` diff --git a/experimental/query/expr/types.go b/experimental/query/expr/types.go index 22cfbc2ff..4765a776e 100644 --- a/experimental/query/expr/types.go +++ b/experimental/query/expr/types.go @@ -36,8 +36,8 @@ func (*QueyHandler) QueryTypeField() string { } // QueryTypes implements query.TypedQueryHandler. -func (*QueyHandler) QueryTypes() []query.QueryTypeDefinition { - return []query.QueryTypeDefinition{} +func (*QueyHandler) QueryTypeDefinitions() []query.QueryTypeDefinitionSpec { + return []query.QueryTypeDefinitionSpec{} } // ReadQuery implements query.TypedQueryHandler. diff --git a/experimental/query/expr/types.json b/experimental/query/expr/types.json new file mode 100644 index 000000000..e95c9a3d7 --- /dev/null +++ b/experimental/query/expr/types.json @@ -0,0 +1,113 @@ +[ + { + "name": "math", + "versions": [ + { + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr/math-query", + "properties": { + "expression": { + "type": "string", + "minLength": 1, + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression" + ] + } + } + ] + }, + { + "name": "reduce", + "versions": [ + { + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr/reduce-query", + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "description": "The reducer" + }, + "settings": { + "properties": { + "mode": { + "type": "string", + "description": "Non-number reduce behavior" + }, + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "mode" + ], + "description": "Reducer Options" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings" + ] + } + } + ] + }, + { + "name": "resample", + "versions": [ + { + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr/resample-query", + "properties": { + "expression": { + "type": "string", + "description": "The math expression" + }, + "window": { + "type": "string", + "description": "The math expression" + }, + "downsampler": { + "type": "string", + "description": "The reducer" + }, + "upsampler": { + "type": "string", + "description": "The reducer" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler" + ], + "description": "QueryType = resample" + } + } + ] + } +] \ No newline at end of file diff --git a/experimental/query/expr/types_test.go b/experimental/query/expr/types_test.go index d9f8f6110..4318aa4c2 100644 --- a/experimental/query/expr/types_test.go +++ b/experimental/query/expr/types_test.go @@ -2,11 +2,12 @@ package expr import ( "encoding/json" - "fmt" + "os" "reflect" "testing" "github.com/grafana/grafana-plugin-sdk-go/experimental/query/schema" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,7 +16,6 @@ func TestQueryTypeDefinitions(t *testing.T) { schema.BuilderOptions{ BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", CodePath: "./", - PluginIDs: []string{"expr"}, }, schema.QueryTypeInfo{ QueryType: string(QueryTypeMath), @@ -32,11 +32,32 @@ func TestQueryTypeDefinitions(t *testing.T) { require.NoError(t, err) // The full schema - s, err := builder.GetFullQuerySchema() + defs, err := builder.QueryTypeDefinitions() require.NoError(t, err) - - out, err := json.MarshalIndent(s, "", " ") + out, err := json.MarshalIndent(defs, "", " ") require.NoError(t, err) - fmt.Printf("%s\n", out) + update := false + outfile := "types.json" + body, err := os.ReadFile(outfile) + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true + } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0644) + require.NoError(t, err, "error writing file") + } + + // // The full schema + // s, err := builder.FullQuerySchema() + // require.NoError(t, err) + + // out, err := json.MarshalIndent(s, "", " ") + // require.NoError(t, err) + + // fmt.Printf("%s\n", out) } diff --git a/experimental/query/schema/builder.go b/experimental/query/schema/builder.go index bfec66ad1..6d7384b7a 100644 --- a/experimental/query/schema/builder.go +++ b/experimental/query/schema/builder.go @@ -19,8 +19,8 @@ type QueryTypeInfo struct { type QueryTypeBuilder struct { opts BuilderOptions reflector *jsonschema.Reflector // Needed to use comments - byType map[string]*query.QueryTypeDefinition - types []*query.QueryTypeDefinition + byType map[string]*query.QueryTypeDefinitionSpec + types []*query.QueryTypeDefinitionSpec } func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { @@ -30,9 +30,10 @@ func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { } def, ok := b.byType[info.QueryType] if !ok { - def = &query.QueryTypeDefinition{ - Name: info.QueryType, - Versions: []query.QueryTypeVersion{}, + def = &query.QueryTypeDefinitionSpec{ + Name: info.QueryType, + DiscriminatorField: b.opts.DiscriminatorField, + Versions: []query.QueryTypeVersion{}, } b.byType[info.QueryType] = def b.types = append(b.types, def) @@ -53,20 +54,18 @@ type BuilderOptions struct { // queryType DiscriminatorField string - - // org-xyz-datasource - PluginIDs []string } func NewBuilder(opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder, error) { r := new(jsonschema.Reflector) + r.DoNotReference = true if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { return nil, err } b := &QueryTypeBuilder{ opts: opts, reflector: r, - byType: make(map[string]*query.QueryTypeDefinition), + byType: make(map[string]*query.QueryTypeDefinitionSpec), } for _, input := range inputs { err := b.Add(input) @@ -77,7 +76,16 @@ func NewBuilder(opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder return b, nil } -func (b *QueryTypeBuilder) GetFullQuerySchema() (*jsonschema.Schema, error) { +func (b *QueryTypeBuilder) QueryTypeDefinitions() (rsp []query.QueryTypeDefinitionSpec, err error) { + for _, v := range b.types { + if v != nil { + rsp = append(rsp, *v) + } + } + return +} + +func (b *QueryTypeBuilder) FullQuerySchema() (*jsonschema.Schema, error) { discriminator := b.opts.DiscriminatorField if discriminator == "" { discriminator = "queryType" From 556b81690010299581a83a96cbc723ee1737af4d Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 12 Feb 2024 19:10:29 -0800 Subject: [PATCH 03/71] no refs --- experimental/query/common.jsonschema | 123 +++++++++++---------------- experimental/query/common_test.go | 7 +- 2 files changed, 55 insertions(+), 75 deletions(-) diff --git a/experimental/query/common.jsonschema b/experimental/query/common.jsonschema index 4f3a02d34..617e55dfe 100644 --- a/experimental/query/common.jsonschema +++ b/experimental/query/common.jsonschema @@ -1,80 +1,24 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/common-query-properties", - "$ref": "#/$defs/CommonQueryProperties", - "$defs": { - "CommonQueryProperties": { - "properties": { - "refId": { - "type": "string", - "description": "RefID is the unique identifier of the query, set by the frontend call." - }, - "resultAssertions": { - "$ref": "#/$defs/ResultAssertions", - "description": "Optionally define expected query result behavior" - }, - "timeRange": { - "$ref": "#/$defs/TimeRange", - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query" - }, - "datasource": { - "$ref": "#/$defs/DataSourceRef", - "description": "The datasource" - }, - "queryType": { - "type": "string", - "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." - }, - "maxDataPoints": { - "type": "integer", - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query." - }, - "intervalMs": { - "type": "number", - "description": "Interval is the suggested duration between time points in a time series query." - }, - "hide": { - "type": "boolean", - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNote this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" - } - }, - "additionalProperties": false, - "type": "object" + "properties": { + "refId": { + "type": "string", + "description": "RefID is the unique identifier of the query, set by the frontend call." }, - "DataSourceRef": { - "properties": { - "type": { - "type": "string", - "description": "The datasource plugin type" - }, - "uid": { - "type": "string", - "description": "Datasource UID" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type", - "uid" - ] - }, - "FrameTypeVersion": { - "items": { - "type": "integer" - }, - "type": "array", - "maxItems": 2, - "minItems": 2 - }, - "ResultAssertions": { + "resultAssertions": { "properties": { "type": { "type": "string", "description": "Type asserts that the frame matches a known type structure." }, "typeVersion": { - "$ref": "#/$defs/FrameTypeVersion", + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2, "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." }, "maxBytes": { @@ -91,9 +35,9 @@ "required": [ "typeVersion" ], - "description": "ResultAssertions define the expected response shape and query behavior." + "description": "Optionally define expected query result behavior" }, - "TimeRange": { + "timeRange": { "properties": { "from": { "type": "string", @@ -110,7 +54,44 @@ "from", "to" ], - "description": "TimeRange represents a time range for a query and is a property of DataQuery." + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query" + }, + "datasource": { + "properties": { + "type": { + "type": "string", + "description": "The datasource plugin type" + }, + "uid": { + "type": "string", + "description": "Datasource UID" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type", + "uid" + ], + "description": "The datasource" + }, + "queryType": { + "type": "string", + "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." + }, + "maxDataPoints": { + "type": "integer", + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query." + }, + "intervalMs": { + "type": "number", + "description": "Interval is the suggested duration between time points in a time series query." + }, + "hide": { + "type": "boolean", + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNote this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" } - } + }, + "additionalProperties": false, + "type": "object" } \ No newline at end of file diff --git a/experimental/query/common_test.go b/experimental/query/common_test.go index 8932de46d..e53da0b0d 100644 --- a/experimental/query/common_test.go +++ b/experimental/query/common_test.go @@ -12,15 +12,14 @@ import ( func TestCommonSupport(t *testing.T) { r := new(jsonschema.Reflector) + r.DoNotReference = true err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/query", "./") require.NoError(t, err) query := r.Reflect(&CommonQueryProperties{}) - common, ok := query.Definitions["CommonQueryProperties"] - require.True(t, ok) - // Hide this old property - common.Properties.Delete("datasourceId") + // // Hide this old property + query.Properties.Delete("datasourceId") out, err := json.MarshalIndent(query, "", " ") require.NoError(t, err) From fa45a9035c94e6e9eda679a1780b11d89892d7be Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 12 Feb 2024 21:12:38 -0800 Subject: [PATCH 04/71] add k8s placeholder --- experimental/query/definition.go | 16 +++++++++++++++- experimental/query/expr/types.go | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/experimental/query/definition.go b/experimental/query/definition.go index 25bdcd976..b0be18b33 100644 --- a/experimental/query/definition.go +++ b/experimental/query/definition.go @@ -12,7 +12,7 @@ type TypedQueryHandler[Q any] interface { QueryTypeField() string // Possible query types (if any) - QueryTypeDefinitions() []QueryTypeDefinitionSpec + QueryTypeDefinitions() []QueryTypeDefinition // Get the query parser for a query type // The version is split from the end of the discriminator field @@ -26,6 +26,20 @@ type TypedQueryHandler[Q any] interface { ) (Q, error) } +// K8s placeholder +type QueryTypeDefinition struct { + Metadata ObjectMeta `json:"metadata"` + + Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` +} + +type ObjectMeta struct { + Name string `json:"name"` + ResourceVersion string `json:"resourceVersion,omitempty"` + CreationTimestamp string `json:"creationTimestamp,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + type QueryTypeDefinitionSpec struct { // The query type value // NOTE: this must be a k8s compatible name diff --git a/experimental/query/expr/types.go b/experimental/query/expr/types.go index 4765a776e..6ce7996fc 100644 --- a/experimental/query/expr/types.go +++ b/experimental/query/expr/types.go @@ -36,8 +36,8 @@ func (*QueyHandler) QueryTypeField() string { } // QueryTypes implements query.TypedQueryHandler. -func (*QueyHandler) QueryTypeDefinitions() []query.QueryTypeDefinitionSpec { - return []query.QueryTypeDefinitionSpec{} +func (*QueyHandler) QueryTypeDefinitions() []query.QueryTypeDefinition { + return []query.QueryTypeDefinition{} } // ReadQuery implements query.TypedQueryHandler. From 645173e3f633c52d987b3ca842ad8e0b34b7ac09 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 12 Feb 2024 23:25:56 -0800 Subject: [PATCH 05/71] parallel k8s --- experimental/query/definition.go | 29 ++-- experimental/query/expr/types.go | 12 +- experimental/query/expr/types.json | 222 ++++++++++++++------------ experimental/query/expr/types_test.go | 35 +--- experimental/query/schema/builder.go | 88 +++++++++- 5 files changed, 227 insertions(+), 159 deletions(-) diff --git a/experimental/query/definition.go b/experimental/query/definition.go index b0be18b33..bc04610e9 100644 --- a/experimental/query/definition.go +++ b/experimental/query/definition.go @@ -1,18 +1,13 @@ package query import ( + "encoding/json" + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" ) type TypedQueryHandler[Q any] interface { - // QueryTypeField is typically "queryType", but may use a different field to - // discriminate different field types. When multiple versions for a field exist, - // The version identifier is appended to the queryType value after a slash. - // eg: queryType=showLabels/v2 - QueryTypeField() string - - // Possible query types (if any) - QueryTypeDefinitions() []QueryTypeDefinition + QueryTypeDefinitionsJSON() (json.RawMessage, error) // Get the query parser for a query type // The version is split from the end of the discriminator field @@ -27,17 +22,25 @@ type TypedQueryHandler[Q any] interface { } // K8s placeholder +// This will serialize to the same byte array, but does not require all the imports type QueryTypeDefinition struct { - Metadata ObjectMeta `json:"metadata"` + ObjectMeta ObjectMeta `json:"metadata,omitempty"` Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` } +// K8s placeholder +// This will serialize to the same byte array, but does not require all the imports +type QueryTypeDefinitionList struct { + ObjectMeta ObjectMeta `json:"metadata,omitempty"` + Items []QueryTypeDefinition `json:"items"` +} + +// K8s placeholder type ObjectMeta struct { - Name string `json:"name"` - ResourceVersion string `json:"resourceVersion,omitempty"` - CreationTimestamp string `json:"creationTimestamp,omitempty"` - Annotations map[string]string `json:"annotations,omitempty"` + Name string `json:"name,omitempty"` // missing on lists + ResourceVersion string `json:"resourceVersion,omitempty"` // indicates something changed + CreationTimestamp string `json:"creationTimestamp,omitempty"` } type QueryTypeDefinitionSpec struct { diff --git a/experimental/query/expr/types.go b/experimental/query/expr/types.go index 6ce7996fc..57924afe5 100644 --- a/experimental/query/expr/types.go +++ b/experimental/query/expr/types.go @@ -1,6 +1,8 @@ package expr import ( + "embed" + "encoding/json" "fmt" "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" @@ -31,13 +33,11 @@ var _ query.TypedQueryHandler[ExpressionQuery] = (*QueyHandler)(nil) type QueyHandler struct{} -func (*QueyHandler) QueryTypeField() string { - return "queryType" -} +//go:embed types.json +var f embed.FS -// QueryTypes implements query.TypedQueryHandler. -func (*QueyHandler) QueryTypeDefinitions() []query.QueryTypeDefinition { - return []query.QueryTypeDefinition{} +func (*QueyHandler) QueryTypeDefinitionsJSON() (json.RawMessage, error) { + return f.ReadFile("types.json") } // ReadQuery implements query.TypedQueryHandler. diff --git a/experimental/query/expr/types.json b/experimental/query/expr/types.json index e95c9a3d7..85b88f145 100644 --- a/experimental/query/expr/types.json +++ b/experimental/query/expr/types.json @@ -1,113 +1,133 @@ -[ - { - "name": "math", - "versions": [ - { - "schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr/math-query", - "properties": { - "expression": { - "type": "string", - "minLength": 1, - "description": "General math expression", - "examples": [ - "$A + 1", - "$A/$B" - ] +{ + "metadata": { + "resourceVersion": "1707808829418" + }, + "items": [ + { + "metadata": { + "name": "math", + "resourceVersion": "1707808587680", + "creationTimestamp": "2024-02-13T07:16:27Z" + }, + "spec": { + "name": "math", + "versions": [ + { + "schema": { + "additionalProperties": false, + "properties": { + "expression": { + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ], + "minLength": 1, + "type": "string" + } + }, + "required": [ + "expression" + ], + "type": "object" } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression" - ] - } + } + ] } - ] - }, - { - "name": "reduce", - "versions": [ - { - "schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr/reduce-query", - "properties": { - "expression": { - "type": "string", - "description": "Reference to other query results" - }, - "reducer": { - "type": "string", - "description": "The reducer" - }, - "settings": { + }, + { + "metadata": { + "name": "reduce", + "resourceVersion": "1707808587680", + "creationTimestamp": "2024-02-13T07:16:27Z" + }, + "spec": { + "name": "reduce", + "versions": [ + { + "schema": { + "additionalProperties": false, "properties": { - "mode": { - "type": "string", - "description": "Non-number reduce behavior" + "expression": { + "description": "Reference to other query results", + "type": "string" }, - "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" + "reducer": { + "description": "The reducer", + "type": "string" + }, + "settings": { + "additionalProperties": false, + "description": "Reducer Options", + "properties": { + "mode": { + "description": "Non-number reduce behavior", + "type": "string" + }, + "replaceWithValue": { + "description": "Only valid when mode is replace", + "type": "number" + } + }, + "required": [ + "mode" + ], + "type": "object" } }, - "additionalProperties": false, - "type": "object", "required": [ - "mode" + "expression", + "reducer", + "settings" ], - "description": "Reducer Options" + "type": "object" } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "reducer", - "settings" - ] - } + } + ] } - ] - }, - { - "name": "resample", - "versions": [ - { - "schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr/resample-query", - "properties": { - "expression": { - "type": "string", - "description": "The math expression" - }, - "window": { - "type": "string", - "description": "The math expression" - }, - "downsampler": { - "type": "string", - "description": "The reducer" - }, - "upsampler": { - "type": "string", - "description": "The reducer" + }, + { + "metadata": { + "name": "resample", + "resourceVersion": "1707808587680", + "creationTimestamp": "2024-02-13T07:16:27Z" + }, + "spec": { + "name": "resample", + "versions": [ + { + "schema": { + "additionalProperties": false, + "description": "QueryType = resample", + "properties": { + "downsampler": { + "description": "The reducer", + "type": "string" + }, + "expression": { + "description": "The math expression", + "type": "string" + }, + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "The math expression", + "type": "string" + } + }, + "required": [ + "expression", + "window", + "downsampler", + "upsampler" + ], + "type": "object" } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler" - ], - "description": "QueryType = resample" - } + } + ] } - ] - } -] \ No newline at end of file + } + ] +} \ No newline at end of file diff --git a/experimental/query/expr/types_test.go b/experimental/query/expr/types_test.go index 4318aa4c2..6f9e908c0 100644 --- a/experimental/query/expr/types_test.go +++ b/experimental/query/expr/types_test.go @@ -1,18 +1,15 @@ package expr import ( - "encoding/json" - "os" "reflect" "testing" "github.com/grafana/grafana-plugin-sdk-go/experimental/query/schema" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestQueryTypeDefinitions(t *testing.T) { - builder, err := schema.NewBuilder( + builder, err := schema.NewBuilder(t, schema.BuilderOptions{ BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", CodePath: "./", @@ -31,33 +28,5 @@ func TestQueryTypeDefinitions(t *testing.T) { }) require.NoError(t, err) - // The full schema - defs, err := builder.QueryTypeDefinitions() - require.NoError(t, err) - out, err := json.MarshalIndent(defs, "", " ") - require.NoError(t, err) - - update := false - outfile := "types.json" - body, err := os.ReadFile(outfile) - if err == nil { - if !assert.JSONEq(t, string(out), string(body)) { - update = true - } - } else { - update = true - } - if update { - err = os.WriteFile(outfile, out, 0644) - require.NoError(t, err, "error writing file") - } - - // // The full schema - // s, err := builder.FullQuerySchema() - // require.NoError(t, err) - - // out, err := json.MarshalIndent(s, "", " ") - // require.NoError(t, err) - - // fmt.Printf("%s\n", out) + _ = builder.Write("types.json") } diff --git a/experimental/query/schema/builder.go b/experimental/query/schema/builder.go index 6d7384b7a..ac4c79f8b 100644 --- a/experimental/query/schema/builder.go +++ b/experimental/query/schema/builder.go @@ -3,11 +3,16 @@ package schema import ( "encoding/json" "fmt" + "os" "reflect" "strings" + "testing" + "time" "github.com/grafana/grafana-plugin-sdk-go/experimental/query" "github.com/invopop/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type QueryTypeInfo struct { @@ -17,6 +22,7 @@ type QueryTypeInfo struct { } type QueryTypeBuilder struct { + t *testing.T opts BuilderOptions reflector *jsonschema.Reflector // Needed to use comments byType map[string]*query.QueryTypeDefinitionSpec @@ -28,6 +34,12 @@ func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { if schema == nil { return fmt.Errorf("missing schema") } + + // Ignored by k8s anyway + schema.Version = "" + schema.ID = "" + schema.Anchor = "" + def, ok := b.byType[info.QueryType] if !ok { def = &query.QueryTypeDefinitionSpec{ @@ -56,13 +68,14 @@ type BuilderOptions struct { DiscriminatorField string } -func NewBuilder(opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder, error) { +func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder, error) { r := new(jsonschema.Reflector) r.DoNotReference = true if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { return nil, err } b := &QueryTypeBuilder{ + t: t, opts: opts, reflector: r, byType: make(map[string]*query.QueryTypeDefinitionSpec), @@ -76,13 +89,76 @@ func NewBuilder(opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder return b, nil } -func (b *QueryTypeBuilder) QueryTypeDefinitions() (rsp []query.QueryTypeDefinitionSpec, err error) { - for _, v := range b.types { - if v != nil { - rsp = append(rsp, *v) +func (b *QueryTypeBuilder) Write(outfile string) json.RawMessage { + t := b.t + t.Helper() + + now := time.Now().UTC() + rv := fmt.Sprintf("%d", now.UnixMilli()) + + defs := query.QueryTypeDefinitionList{} + byName := make(map[string]*query.QueryTypeDefinition) + body, err := os.ReadFile(outfile) + if err == nil { + err = json.Unmarshal(body, &defs) + if err == nil { + for i, def := range defs.Items { + byName[def.ObjectMeta.Name] = &defs.Items[i] + } + } + } + + // The updated schemas + for _, spec := range b.types { + found, ok := byName[spec.Name] + if !ok { + defs.ObjectMeta.ResourceVersion = rv + defs.Items = append(defs.Items, query.QueryTypeDefinition{ + ObjectMeta: query.ObjectMeta{ + Name: spec.Name, + ResourceVersion: rv, + CreationTimestamp: now.Format(time.RFC3339), + }, + Spec: *spec, + }) + } else { + var o1, o2 interface{} + b1, _ := json.Marshal(spec) + b2, _ := json.Marshal(found.Spec) + _ = json.Unmarshal(b1, &o1) + _ = json.Unmarshal(b2, &o2) + if !reflect.DeepEqual(o1, o2) { + found.ObjectMeta.ResourceVersion = rv + found.Spec = *spec + } + delete(byName, spec.Name) + } + } + + if defs.ObjectMeta.ResourceVersion == "" { + defs.ObjectMeta.ResourceVersion = rv + } + + if len(byName) > 0 { + require.FailNow(t, "query type removed, manually update (for now)") + } + + out, err := json.MarshalIndent(defs, "", " ") + require.NoError(t, err) + + update := false + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0644) + require.NoError(t, err, "error writing file") } - return + return out } func (b *QueryTypeBuilder) FullQuerySchema() (*jsonschema.Schema, error) { From f6bb32321af84e747579253b83eaa98b112b3ea9 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Feb 2024 14:03:37 -0800 Subject: [PATCH 06/71] now with examples --- experimental/query/definition.go | 10 ++- experimental/query/expr/types.json | 94 ++++++++++++++++++------- experimental/query/expr/types_test.go | 33 +++++++++ experimental/query/schema/builder.go | 79 ++++++++++++++++++++- experimental/query/schema/enums.go | 8 +-- experimental/query/schema/enums_test.go | 2 +- 6 files changed, 193 insertions(+), 33 deletions(-) diff --git a/experimental/query/definition.go b/experimental/query/definition.go index bc04610e9..4715f689c 100644 --- a/experimental/query/definition.go +++ b/experimental/query/definition.go @@ -70,10 +70,18 @@ type QueryTypeVersion struct { Schema any `json:"schema"` // Examples (include a wrapper) ideally a template! - Examples []any `json:"examples,omitempty"` + Examples []QueryExample `json:"examples,omitempty"` // Changelog defines the changed from the previous version // All changes in the same version *must* be backwards compatible // Only notable changes will be shown here, for the full version history see git! Changelog []string `json:"changelog,omitempty"` } + +type QueryExample struct { + // Version identifier or empty if only one exists + Name string `json:"name,omitempty"` + + // An example query + Query any `json:"query"` +} diff --git a/experimental/query/expr/types.json b/experimental/query/expr/types.json index 85b88f145..124bdedcd 100644 --- a/experimental/query/expr/types.json +++ b/experimental/query/expr/types.json @@ -6,7 +6,7 @@ { "metadata": { "name": "math", - "resourceVersion": "1707808587680", + "resourceVersion": "1707861521526", "creationTimestamp": "2024-02-13T07:16:27Z" }, "spec": { @@ -14,23 +14,37 @@ "versions": [ { "schema": { - "additionalProperties": false, "properties": { "expression": { + "type": "string", + "minLength": 1, "description": "General math expression", "examples": [ "$A + 1", "$A/$B" - ], - "minLength": 1, - "type": "string" + ] } }, + "additionalProperties": false, + "type": "object", "required": [ "expression" - ], - "type": "object" - } + ] + }, + "examples": [ + { + "name": "constant addition", + "query": { + "expression": "$A + 10" + } + }, + { + "name": "math with two queries", + "query": { + "expression": "$A - $B" + } + } + ] } ] } @@ -38,7 +52,7 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1707808587680", + "resourceVersion": "1707861521526", "creationTimestamp": "2024-02-13T07:16:27Z" }, "spec": { @@ -46,42 +60,74 @@ "versions": [ { "schema": { - "additionalProperties": false, "properties": { "expression": { - "description": "Reference to other query results", - "type": "string" + "type": "string", + "description": "Reference to other query results" }, "reducer": { - "description": "The reducer", - "type": "string" + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" + } }, "settings": { - "additionalProperties": false, - "description": "Reducer Options", "properties": { "mode": { - "description": "Non-number reduce behavior", - "type": "string" + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "x-enum-description": { + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" + } }, "replaceWithValue": { - "description": "Only valid when mode is replace", - "type": "number" + "type": "number", + "description": "Only valid when mode is replace" } }, + "additionalProperties": false, + "type": "object", "required": [ "mode" ], - "type": "object" + "description": "Reducer Options" } }, + "additionalProperties": false, + "type": "object", "required": [ "expression", "reducer", "settings" - ], - "type": "object" - } + ] + }, + "examples": [ + { + "name": "get max value", + "query": { + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + } + ] } ] } diff --git a/experimental/query/expr/types_test.go b/experimental/query/expr/types_test.go index 6f9e908c0..8b8ff14a8 100644 --- a/experimental/query/expr/types_test.go +++ b/experimental/query/expr/types_test.go @@ -4,6 +4,7 @@ import ( "reflect" "testing" + "github.com/grafana/grafana-plugin-sdk-go/experimental/query" "github.com/grafana/grafana-plugin-sdk-go/experimental/query/schema" "github.com/stretchr/testify/require" ) @@ -13,14 +14,46 @@ func TestQueryTypeDefinitions(t *testing.T) { schema.BuilderOptions{ BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", CodePath: "./", + // We need to identify the enum fields explicitly :( + // *AND* have the +enum common for this to work + Enums: []reflect.Type{ + reflect.TypeOf(ReducerSum), // pick an example value (not the root) + reflect.TypeOf(ReduceModeDrop), // pick an example value (not the root) + }, }, schema.QueryTypeInfo{ QueryType: string(QueryTypeMath), GoType: reflect.TypeOf(&MathQuery{}), + Examples: []query.QueryExample{ + { + Name: "constant addition", + Query: MathQuery{ + Expression: "$A + 10", + }, + }, + { + Name: "math with two queries", + Query: MathQuery{ + Expression: "$A - $B", + }, + }, + }, }, schema.QueryTypeInfo{ QueryType: string(QueryTypeReduce), GoType: reflect.TypeOf(&ReduceQuery{}), + Examples: []query.QueryExample{ + { + Name: "get max value", + Query: ReduceQuery{ + Expression: "$A", + Reducer: ReducerMax, + Settings: ReduceSettings{ + Mode: ReduceModeDrop, + }, + }, + }, + }, }, schema.QueryTypeInfo{ QueryType: string(QueryTypeResample), diff --git a/experimental/query/schema/builder.go b/experimental/query/schema/builder.go index ac4c79f8b..e8e4ad433 100644 --- a/experimental/query/schema/builder.go +++ b/experimental/query/schema/builder.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "reflect" + "regexp" "strings" "testing" "time" @@ -19,6 +20,7 @@ type QueryTypeInfo struct { QueryType string Version string GoType reflect.Type + Examples []query.QueryExample } type QueryTypeBuilder struct { @@ -35,6 +37,8 @@ func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { return fmt.Errorf("missing schema") } + b.enumify(schema) + // Ignored by k8s anyway schema.Version = "" schema.ID = "" @@ -51,8 +55,9 @@ func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { b.types = append(b.types, def) } def.Versions = append(def.Versions, query.QueryTypeVersion{ - Version: info.Version, - Schema: schema, + Version: info.Version, + Schema: schema, + Examples: info.Examples, }) return nil } @@ -66,6 +71,9 @@ type BuilderOptions struct { // queryType DiscriminatorField string + + // explicitly define the enumeration fields + Enums []reflect.Type } func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder, error) { @@ -74,6 +82,38 @@ func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*Qu if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { return nil, err } + if len(opts.Enums) > 0 { + fields, err := findEnumFields(opts.BasePackage, opts.CodePath) + if err != nil { + return nil, err + } + enumMapper := map[reflect.Type]*jsonschema.Schema{} + for _, etype := range opts.Enums { + for _, f := range fields { + if f.Name == etype.Name() && f.Package == etype.PkgPath() { + enumValueDescriptions := map[string]string{} + s := &jsonschema.Schema{ + Type: "string", + Extras: map[string]any{ + "x-enum-description": enumValueDescriptions, + }, + } + for _, val := range f.Values { + s.Enum = append(s.Enum, val.Value) + if val.Comment != "" { + enumValueDescriptions[val.Value] = val.Comment + } + } + enumMapper[etype] = s + } + } + } + + r.Mapper = func(t reflect.Type) *jsonschema.Schema { + return enumMapper[t] + } + } + b := &QueryTypeBuilder{ t: t, opts: opts, @@ -89,6 +129,41 @@ func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*Qu return b, nil } +// whitespaceRegex is the regex for consecutive whitespaces. +var whitespaceRegex = regexp.MustCompile(`\s+`) + +func (b *QueryTypeBuilder) enumify(s *jsonschema.Schema) { + if len(s.Enum) > 0 && s.Extras != nil { + extra, ok := s.Extras["x-enum-description"] + if !ok { + return + } + + lookup, ok := extra.(map[string]string) + if !ok { + return + } + + lines := []string{} + if s.Description != "" { + lines = append(lines, s.Description, "\n") + } + lines = append(lines, "Possible enum values:") + for _, v := range s.Enum { + c := lookup[v.(string)] + c = whitespaceRegex.ReplaceAllString(c, " ") + lines = append(lines, fmt.Sprintf(" - `%q` %s", v, c)) + } + + s.Description = strings.Join(lines, "\n") + return + } + + for pair := s.Properties.Oldest(); pair != nil; pair = pair.Next() { + b.enumify(pair.Value) + } +} + func (b *QueryTypeBuilder) Write(outfile string) json.RawMessage { t := b.t t.Helper() diff --git a/experimental/query/schema/enums.go b/experimental/query/schema/enums.go index f3f5e1d1b..8fa9d4cc9 100644 --- a/experimental/query/schema/enums.go +++ b/experimental/query/schema/enums.go @@ -1,7 +1,6 @@ package schema import ( - "fmt" "io/fs" gopath "path" "path/filepath" @@ -25,7 +24,7 @@ type EnumField struct { Values []EnumValue } -func FindEnumFields(base, path string) ([]EnumField, error) { +func findEnumFields(base, path string) ([]EnumField, error) { fset := token.NewFileSet() dict := make(map[string][]*ast.Package) err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { @@ -76,8 +75,7 @@ func FindEnumFields(base, path string) ([]EnumField, error) { Comment: strings.TrimSpace(strings.TrimSuffix(txt, "+enum")), }) field = &fields[len(fields)-1] - - fmt.Printf("ENUM: %s.%s // %s\n", pkg, typ, txt) + //fmt.Printf("ENUM: %s.%s // %s\n", pkg, typ, txt) } } case *ast.ValueSpec: @@ -93,7 +91,7 @@ func FindEnumFields(base, path string) ([]EnumField, error) { val := strings.TrimPrefix(v.Value, `"`) val = strings.TrimSuffix(val, `"`) txt = strings.TrimSpace(txt) - fmt.Printf("%s // %s // %s\n", typ, val, txt) + //fmt.Printf("%s // %s // %s\n", typ, val, txt) field.Values = append(field.Values, EnumValue{ Value: val, Comment: txt, diff --git a/experimental/query/schema/enums_test.go b/experimental/query/schema/enums_test.go index 1d1442e8a..b5b751d3a 100644 --- a/experimental/query/schema/enums_test.go +++ b/experimental/query/schema/enums_test.go @@ -9,7 +9,7 @@ import ( ) func TestFindEnums(t *testing.T) { - fields, err := FindEnumFields( + fields, err := findEnumFields( "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", "../expr") require.NoError(t, err) From e2341d6a4b2481e0564da03800e700443cf86ee8 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Feb 2024 15:01:46 -0800 Subject: [PATCH 07/71] ok with dataframe now --- experimental/query/schema/builder.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/experimental/query/schema/builder.go b/experimental/query/schema/builder.go index e8e4ad433..3a3a8e9de 100644 --- a/experimental/query/schema/builder.go +++ b/experimental/query/schema/builder.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/experimental/query" "github.com/invopop/jsonschema" "github.com/stretchr/testify/assert" @@ -82,12 +83,24 @@ func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*Qu if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { return nil, err } + customMapper := map[reflect.Type]*jsonschema.Schema{ + reflect.TypeOf(data.Frame{}): { + Type: "object", + Extras: map[string]any{ + "x-grafana-type": "data.DataFrame", + }, + AdditionalProperties: jsonschema.TrueSchema, + }, + } + r.Mapper = func(t reflect.Type) *jsonschema.Schema { + return customMapper[t] + } + if len(opts.Enums) > 0 { fields, err := findEnumFields(opts.BasePackage, opts.CodePath) if err != nil { return nil, err } - enumMapper := map[reflect.Type]*jsonschema.Schema{} for _, etype := range opts.Enums { for _, f := range fields { if f.Name == etype.Name() && f.Package == etype.PkgPath() { @@ -104,14 +117,10 @@ func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*Qu enumValueDescriptions[val.Value] = val.Comment } } - enumMapper[etype] = s + customMapper[etype] = s } } } - - r.Mapper = func(t reflect.Type) *jsonschema.Schema { - return enumMapper[t] - } } b := &QueryTypeBuilder{ From 992463de0282426fb711537b0dbe65a9ba628af5 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Feb 2024 15:05:53 -0800 Subject: [PATCH 08/71] ok with dataframe now --- experimental/query/expr/resample.go | 6 ++- experimental/query/expr/types.json | 78 ++++++++++++++++------------- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/experimental/query/expr/resample.go b/experimental/query/expr/resample.go index d28609134..4c099dfbd 100644 --- a/experimental/query/expr/resample.go +++ b/experimental/query/expr/resample.go @@ -1,11 +1,13 @@ package expr +import "github.com/grafana/grafana-plugin-sdk-go/data" + // QueryType = resample type ResampleQuery struct { // The math expression Expression string `json:"expression"` - // The math expression + // A time duration string Window string `json:"window"` // The reducer @@ -13,6 +15,8 @@ type ResampleQuery struct { // The reducer Upsampler string `json:"upsampler"` + + LoadedDimensions *data.Frame `json:"loadedDimensions"` } func (*ResampleQuery) ExpressionQueryType() QueryType { diff --git a/experimental/query/expr/types.json b/experimental/query/expr/types.json index 124bdedcd..515b5a7e1 100644 --- a/experimental/query/expr/types.json +++ b/experimental/query/expr/types.json @@ -14,22 +14,22 @@ "versions": [ { "schema": { + "additionalProperties": false, "properties": { "expression": { - "type": "string", - "minLength": 1, "description": "General math expression", "examples": [ "$A + 1", "$A/$B" - ] + ], + "minLength": 1, + "type": "string" } }, - "additionalProperties": false, - "type": "object", "required": [ "expression" - ] + ], + "type": "object" }, "examples": [ { @@ -60,13 +60,14 @@ "versions": [ { "schema": { + "additionalProperties": false, "properties": { "expression": { - "type": "string", - "description": "Reference to other query results" + "description": "Reference to other query results", + "type": "string" }, "reducer": { - "type": "string", + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", "enum": [ "sum", "mean", @@ -75,46 +76,45 @@ "count", "last" ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "type": "string", "x-enum-description": { "mean": "The mean", "sum": "The sum" } }, "settings": { + "additionalProperties": false, + "description": "Reducer Options", "properties": { "mode": { - "type": "string", + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", "enum": [ "dropNN", "replaceNN" ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "type": "string", "x-enum-description": { "dropNN": "Drop non-numbers", "replaceNN": "Replace non-numbers" } }, "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" + "description": "Only valid when mode is replace", + "type": "number" } }, - "additionalProperties": false, - "type": "object", "required": [ "mode" ], - "description": "Reducer Options" + "type": "object" } }, - "additionalProperties": false, - "type": "object", "required": [ "expression", "reducer", "settings" - ] + ], + "type": "object" }, "examples": [ { @@ -135,7 +135,7 @@ { "metadata": { "name": "resample", - "resourceVersion": "1707808587680", + "resourceVersion": "1707865501262", "creationTimestamp": "2024-02-13T07:16:27Z" }, "spec": { @@ -143,33 +143,39 @@ "versions": [ { "schema": { - "additionalProperties": false, - "description": "QueryType = resample", "properties": { - "downsampler": { - "description": "The reducer", - "type": "string" - }, "expression": { - "description": "The math expression", - "type": "string" + "type": "string", + "description": "The math expression" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, + "downsampler": { + "type": "string", + "description": "The reducer" }, "upsampler": { - "description": "The reducer", - "type": "string" + "type": "string", + "description": "The reducer" }, - "window": { - "description": "The math expression", - "type": "string" + "loadedDimensions": { + "additionalProperties": true, + "type": "object", + "x-grafana-type": "data.DataFrame" } }, + "additionalProperties": false, + "type": "object", "required": [ "expression", "window", "downsampler", - "upsampler" + "upsampler", + "loadedDimensions" ], - "type": "object" + "description": "QueryType = resample" } } ] From 58b32eea279169f1d54eda0a76dc5555fe8a8190 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Feb 2024 17:15:39 -0800 Subject: [PATCH 09/71] no versions --- experimental/query/definition.go | 35 +--- experimental/query/expr/math.go | 2 +- experimental/query/expr/types.go | 8 +- experimental/query/expr/types.json | 271 +++++++++++++------------- experimental/query/expr/types_test.go | 17 +- experimental/query/schema/builder.go | 156 ++++----------- 6 files changed, 192 insertions(+), 297 deletions(-) diff --git a/experimental/query/definition.go b/experimental/query/definition.go index 4715f689c..8b2a0b314 100644 --- a/experimental/query/definition.go +++ b/experimental/query/definition.go @@ -1,19 +1,13 @@ package query import ( - "encoding/json" - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" ) -type TypedQueryHandler[Q any] interface { - QueryTypeDefinitionsJSON() (json.RawMessage, error) - +type TypedQueryReader[Q any] interface { // Get the query parser for a query type // The version is split from the end of the discriminator field ReadQuery( - // The query type split by version (when multiple exist) - queryType string, version string, // Properties that have been parsed off the same node common CommonQueryProperties, // An iterator with context for the full node (include common values) @@ -36,36 +30,27 @@ type QueryTypeDefinitionList struct { Items []QueryTypeDefinition `json:"items"` } -// K8s placeholder +// K8s compatible type ObjectMeta struct { - Name string `json:"name,omitempty"` // missing on lists - ResourceVersion string `json:"resourceVersion,omitempty"` // indicates something changed + // The name is for k8s and description, but not used in the schema + Name string `json:"name,omitempty"` + // Changes indicate that *something * changed + ResourceVersion string `json:"resourceVersion,omitempty"` + // Timestamp CreationTimestamp string `json:"creationTimestamp,omitempty"` } type QueryTypeDefinitionSpec struct { - // The query type value - // NOTE: this must be a k8s compatible name - Name string `json:"name,omitempty"` // must be k8s name? compatible - // DiscriminatorField is the field used to link behavior to this specific // query type. It is typically "queryType", but can be another field if necessary DiscriminatorField string `json:"discriminatorField,omitempty"` + // The discriminator value + DiscriminatorValue string `json:"discriminatorValue,omitempty"` + // Describe whe the query type is for Description string `json:"description,omitempty"` - // Versions (most recent first) - Versions []QueryTypeVersion `json:"versions"` - - // When multiple versions exist, this is the preferredVersion - PreferredVersion string `json:"preferredVersion,omitempty"` -} - -type QueryTypeVersion struct { - // Version identifier or empty if only one exists - Version string `json:"version,omitempty"` - // The JSONSchema definition for the non-common fields Schema any `json:"schema"` diff --git a/experimental/query/expr/math.go b/experimental/query/expr/math.go index ddc42a57c..2f27b7ff4 100644 --- a/experimental/query/expr/math.go +++ b/experimental/query/expr/math.go @@ -20,7 +20,7 @@ func (q *MathQuery) Variables() []string { return q.variables } -func readMathQuery(version string, iter *jsoniter.Iterator) (q *MathQuery, err error) { +func readMathQuery(iter *jsoniter.Iterator) (q *MathQuery, err error) { fname := "" for fname, err = iter.ReadObject(); fname != "" && err == nil; fname, err = iter.ReadObject() { switch fname { diff --git a/experimental/query/expr/types.go b/experimental/query/expr/types.go index 57924afe5..ef43a35b6 100644 --- a/experimental/query/expr/types.go +++ b/experimental/query/expr/types.go @@ -29,7 +29,7 @@ type ExpressionQuery interface { Variables() []string } -var _ query.TypedQueryHandler[ExpressionQuery] = (*QueyHandler)(nil) +var _ query.TypedQueryReader[ExpressionQuery] = (*QueyHandler)(nil) type QueyHandler struct{} @@ -42,17 +42,15 @@ func (*QueyHandler) QueryTypeDefinitionsJSON() (json.RawMessage, error) { // ReadQuery implements query.TypedQueryHandler. func (*QueyHandler) ReadQuery( - // The query type split by version (when multiple exist) - queryType string, version string, // Properties that have been parsed off the same node common query.CommonQueryProperties, // An iterator with context for the full node (include common values) iter *jsoniter.Iterator, ) (ExpressionQuery, error) { - qt := QueryType(queryType) + qt := QueryType(common.QueryType) switch qt { case QueryTypeMath: - return readMathQuery(version, iter) + return readMathQuery(iter) case QueryTypeReduce: q := &ReduceQuery{} diff --git a/experimental/query/expr/types.json b/experimental/query/expr/types.json index 515b5a7e1..a1a390132 100644 --- a/experimental/query/expr/types.json +++ b/experimental/query/expr/types.json @@ -1,50 +1,47 @@ { "metadata": { - "resourceVersion": "1707808829418" + "resourceVersion": "1707873133114" }, "items": [ { "metadata": { "name": "math", - "resourceVersion": "1707861521526", - "creationTimestamp": "2024-02-13T07:16:27Z" + "resourceVersion": "1707873191437", + "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { - "name": "math", - "versions": [ + "discriminatorField": "queryType", + "discriminatorValue": "math", + "schema": { + "properties": { + "expression": { + "type": "string", + "minLength": 1, + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression" + ] + }, + "examples": [ { - "schema": { - "additionalProperties": false, - "properties": { - "expression": { - "description": "General math expression", - "examples": [ - "$A + 1", - "$A/$B" - ], - "minLength": 1, - "type": "string" - } - }, - "required": [ - "expression" - ], - "type": "object" - }, - "examples": [ - { - "name": "constant addition", - "query": { - "expression": "$A + 10" - } - }, - { - "name": "math with two queries", - "query": { - "expression": "$A - $B" - } - } - ] + "name": "constant addition", + "query": { + "expression": "$A + 10" + } + }, + { + "name": "math with two queries", + "query": { + "expression": "$A - $B" + } } ] } @@ -52,82 +49,79 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1707861521526", - "creationTimestamp": "2024-02-13T07:16:27Z" + "resourceVersion": "1707873191437", + "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { - "name": "reduce", - "versions": [ - { - "schema": { - "additionalProperties": false, + "discriminatorField": "queryType", + "discriminatorValue": "reduce", + "schema": { + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" + } + }, + "settings": { "properties": { - "expression": { - "description": "Reference to other query results", - "type": "string" - }, - "reducer": { - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "mode": { + "type": "string", "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" + "dropNN", + "replaceNN" ], - "type": "string", + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", "x-enum-description": { - "mean": "The mean", - "sum": "The sum" + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" } }, - "settings": { - "additionalProperties": false, - "description": "Reducer Options", - "properties": { - "mode": { - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "enum": [ - "dropNN", - "replaceNN" - ], - "type": "string", - "x-enum-description": { - "dropNN": "Drop non-numbers", - "replaceNN": "Replace non-numbers" - } - }, - "replaceWithValue": { - "description": "Only valid when mode is replace", - "type": "number" - } - }, - "required": [ - "mode" - ], - "type": "object" + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" } }, + "additionalProperties": false, + "type": "object", "required": [ - "expression", - "reducer", - "settings" + "mode" ], - "type": "object" - }, - "examples": [ - { - "name": "get max value", - "query": { - "expression": "$A", - "reducer": "max", - "settings": { - "mode": "dropNN" - } - } + "description": "Reducer Options" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings" + ] + }, + "examples": [ + { + "name": "get max value", + "query": { + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" } - ] + } } ] } @@ -135,50 +129,47 @@ { "metadata": { "name": "resample", - "resourceVersion": "1707865501262", - "creationTimestamp": "2024-02-13T07:16:27Z" + "resourceVersion": "1707873191437", + "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { - "name": "resample", - "versions": [ - { - "schema": { - "properties": { - "expression": { - "type": "string", - "description": "The math expression" - }, - "window": { - "type": "string", - "description": "A time duration string" - }, - "downsampler": { - "type": "string", - "description": "The reducer" - }, - "upsampler": { - "type": "string", - "description": "The reducer" - }, - "loadedDimensions": { - "additionalProperties": true, - "type": "object", - "x-grafana-type": "data.DataFrame" - } - }, - "additionalProperties": false, + "discriminatorField": "queryType", + "discriminatorValue": "resample", + "schema": { + "properties": { + "expression": { + "type": "string", + "description": "The math expression" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, + "downsampler": { + "type": "string", + "description": "The reducer" + }, + "upsampler": { + "type": "string", + "description": "The reducer" + }, + "loadedDimensions": { + "additionalProperties": true, "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions" - ], - "description": "QueryType = resample" + "x-grafana-type": "data.DataFrame" } - } - ] + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions" + ], + "description": "QueryType = resample" + } } } ] diff --git a/experimental/query/expr/types_test.go b/experimental/query/expr/types_test.go index 8b8ff14a8..88339030b 100644 --- a/experimental/query/expr/types_test.go +++ b/experimental/query/expr/types_test.go @@ -12,8 +12,9 @@ import ( func TestQueryTypeDefinitions(t *testing.T) { builder, err := schema.NewBuilder(t, schema.BuilderOptions{ - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", - CodePath: "./", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", + DiscriminatorField: "queryType", + CodePath: "./", // We need to identify the enum fields explicitly :( // *AND* have the +enum common for this to work Enums: []reflect.Type{ @@ -22,8 +23,8 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }, schema.QueryTypeInfo{ - QueryType: string(QueryTypeMath), - GoType: reflect.TypeOf(&MathQuery{}), + Discriminator: string(QueryTypeMath), + GoType: reflect.TypeOf(&MathQuery{}), Examples: []query.QueryExample{ { Name: "constant addition", @@ -40,8 +41,8 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }, schema.QueryTypeInfo{ - QueryType: string(QueryTypeReduce), - GoType: reflect.TypeOf(&ReduceQuery{}), + Discriminator: string(QueryTypeReduce), + GoType: reflect.TypeOf(&ReduceQuery{}), Examples: []query.QueryExample{ { Name: "get max value", @@ -56,8 +57,8 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }, schema.QueryTypeInfo{ - QueryType: string(QueryTypeResample), - GoType: reflect.TypeOf(&ResampleQuery{}), + Discriminator: string(QueryTypeResample), + GoType: reflect.TypeOf(&ResampleQuery{}), }) require.NoError(t, err) diff --git a/experimental/query/schema/builder.go b/experimental/query/schema/builder.go index 3a3a8e9de..f87fc10dd 100644 --- a/experimental/query/schema/builder.go +++ b/experimental/query/schema/builder.go @@ -18,18 +18,21 @@ import ( ) type QueryTypeInfo struct { - QueryType string - Version string - GoType reflect.Type - Examples []query.QueryExample + // The management name + Name string + // The discriminator value (requires the field set in ops) + Discriminator string + // Raw GO type used for reflection + GoType reflect.Type + // Add sample queries + Examples []query.QueryExample } type QueryTypeBuilder struct { t *testing.T opts BuilderOptions reflector *jsonschema.Reflector // Needed to use comments - byType map[string]*query.QueryTypeDefinitionSpec - types []*query.QueryTypeDefinitionSpec + defs []query.QueryTypeDefinition } func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { @@ -45,20 +48,28 @@ func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { schema.ID = "" schema.Anchor = "" - def, ok := b.byType[info.QueryType] - if !ok { - def = &query.QueryTypeDefinitionSpec{ - Name: info.QueryType, - DiscriminatorField: b.opts.DiscriminatorField, - Versions: []query.QueryTypeVersion{}, + name := info.Name + if name == "" { + name = info.Discriminator + if name == "" { + return fmt.Errorf("missing name or discriminator") } - b.byType[info.QueryType] = def - b.types = append(b.types, def) } - def.Versions = append(def.Versions, query.QueryTypeVersion{ - Version: info.Version, - Schema: schema, - Examples: info.Examples, + + if info.Discriminator != "" && b.opts.DiscriminatorField == "" { + return fmt.Errorf("missing discriminator field") + } + + b.defs = append(b.defs, query.QueryTypeDefinition{ + ObjectMeta: query.ObjectMeta{ + Name: name, + }, + Spec: query.QueryTypeDefinitionSpec{ + DiscriminatorField: b.opts.DiscriminatorField, + DiscriminatorValue: info.Discriminator, + Schema: schema, + Examples: info.Examples, + }, }) return nil } @@ -127,7 +138,6 @@ func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*Qu t: t, opts: opts, reflector: r, - byType: make(map[string]*query.QueryTypeDefinitionSpec), } for _, input := range inputs { err := b.Add(input) @@ -193,29 +203,25 @@ func (b *QueryTypeBuilder) Write(outfile string) json.RawMessage { } // The updated schemas - for _, spec := range b.types { - found, ok := byName[spec.Name] + for _, def := range b.defs { + found, ok := byName[def.ObjectMeta.Name] if !ok { defs.ObjectMeta.ResourceVersion = rv - defs.Items = append(defs.Items, query.QueryTypeDefinition{ - ObjectMeta: query.ObjectMeta{ - Name: spec.Name, - ResourceVersion: rv, - CreationTimestamp: now.Format(time.RFC3339), - }, - Spec: *spec, - }) + def.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) + + defs.Items = append(defs.Items, def) } else { var o1, o2 interface{} - b1, _ := json.Marshal(spec) + b1, _ := json.Marshal(def.Spec) b2, _ := json.Marshal(found.Spec) _ = json.Unmarshal(b1, &o1) _ = json.Unmarshal(b2, &o2) if !reflect.DeepEqual(o1, o2) { found.ObjectMeta.ResourceVersion = rv - found.Spec = *spec + found.Spec = def.Spec } - delete(byName, spec.Name) + delete(byName, def.ObjectMeta.Name) } } @@ -244,89 +250,3 @@ func (b *QueryTypeBuilder) Write(outfile string) json.RawMessage { } return out } - -func (b *QueryTypeBuilder) FullQuerySchema() (*jsonschema.Schema, error) { - discriminator := b.opts.DiscriminatorField - if discriminator == "" { - discriminator = "queryType" - } - - query, err := asJSONSchema(query.GetCommonJSONSchema()) - if err != nil { - return nil, err - } - query.Ref = "" - common, ok := query.Definitions["CommonQueryProperties"] - if !ok { - return nil, fmt.Errorf("error finding common properties") - } - delete(query.Definitions, "CommonQueryProperties") - - for _, t := range b.types { - for _, v := range t.Versions { - s, err := asJSONSchema(v.Schema) - if err != nil { - return nil, err - } - if s.Ref == "" { - return nil, fmt.Errorf("only ref elements supported right now") - } - - ref := strings.TrimPrefix(s.Ref, "#/$defs/") - body := s - - // Add all types to the - for key, def := range s.Definitions { - if key == ref { - body = def - } else { - query.Definitions[key] = def - } - } - - if body.Properties == nil { - return nil, fmt.Errorf("expected properties on body") - } - - for pair := common.Properties.Oldest(); pair != nil; pair = pair.Next() { - body.Properties.Set(pair.Key, pair.Value) - } - body.Required = append(body.Required, "refId") - - if t.Name != "" { - key := t.Name - if v.Version != "" { - key += "/" + v.Version - } - - p, err := body.Properties.GetAndMoveToFront(discriminator) - if err != nil { - return nil, fmt.Errorf("missing discriminator field: %s", discriminator) - } - p.Const = key - p.Enum = nil - - body.Required = append(body.Required, discriminator) - } - - query.OneOf = append(query.OneOf, body) - } - } - - return query, nil -} - -// Always creates a copy so we can modify it -func asJSONSchema(v any) (*jsonschema.Schema, error) { - var err error - s := &jsonschema.Schema{} - b, ok := v.([]byte) - if !ok { - b, err = json.Marshal(v) - if err != nil { - return nil, err - } - } - err = json.Unmarshal(b, s) - return s, err -} From d791fc906236d5559817517b374c0fe9f29b4ef0 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 13 Feb 2024 20:22:09 -0800 Subject: [PATCH 10/71] add apiVersion --- experimental/query/definition.go | 8 ++- experimental/query/expr/types.json | 74 ++++++++++++++------------- experimental/query/expr/types_test.go | 4 +- experimental/query/schema/builder.go | 14 ++--- 4 files changed, 54 insertions(+), 46 deletions(-) diff --git a/experimental/query/definition.go b/experimental/query/definition.go index 8b2a0b314..4e770a230 100644 --- a/experimental/query/definition.go +++ b/experimental/query/definition.go @@ -26,8 +26,12 @@ type QueryTypeDefinition struct { // K8s placeholder // This will serialize to the same byte array, but does not require all the imports type QueryTypeDefinitionList struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` - Items []QueryTypeDefinition `json:"items"` + Kind string `json:"kind"` // "QueryTypeDefinitionList", + ApiVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", + + ObjectMeta `json:"metadata,omitempty"` + + Items []QueryTypeDefinition `json:"items"` } // K8s compatible diff --git a/experimental/query/expr/types.json b/experimental/query/expr/types.json index a1a390132..1621655fd 100644 --- a/experimental/query/expr/types.json +++ b/experimental/query/expr/types.json @@ -1,4 +1,6 @@ { + "kind": "QueryTypeDefinitionList", + "apiVersion": "query.grafana.app/v0alpha1", "metadata": { "resourceVersion": "1707873133114" }, @@ -13,22 +15,22 @@ "discriminatorField": "queryType", "discriminatorValue": "math", "schema": { + "additionalProperties": false, "properties": { "expression": { - "type": "string", - "minLength": 1, "description": "General math expression", "examples": [ "$A + 1", "$A/$B" - ] + ], + "minLength": 1, + "type": "string" } }, - "additionalProperties": false, - "type": "object", "required": [ "expression" - ] + ], + "type": "object" }, "examples": [ { @@ -56,13 +58,14 @@ "discriminatorField": "queryType", "discriminatorValue": "reduce", "schema": { + "additionalProperties": false, "properties": { "expression": { - "type": "string", - "description": "Reference to other query results" + "description": "Reference to other query results", + "type": "string" }, "reducer": { - "type": "string", + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", "enum": [ "sum", "mean", @@ -71,46 +74,45 @@ "count", "last" ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "type": "string", "x-enum-description": { "mean": "The mean", "sum": "The sum" } }, "settings": { + "additionalProperties": false, + "description": "Reducer Options", "properties": { "mode": { - "type": "string", + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", "enum": [ "dropNN", "replaceNN" ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "type": "string", "x-enum-description": { "dropNN": "Drop non-numbers", "replaceNN": "Replace non-numbers" } }, "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" + "description": "Only valid when mode is replace", + "type": "number" } }, - "additionalProperties": false, - "type": "object", "required": [ "mode" ], - "description": "Reducer Options" + "type": "object" } }, - "additionalProperties": false, - "type": "object", "required": [ "expression", "reducer", "settings" - ] + ], + "type": "object" }, "examples": [ { @@ -136,31 +138,31 @@ "discriminatorField": "queryType", "discriminatorValue": "resample", "schema": { + "additionalProperties": false, + "description": "QueryType = resample", "properties": { - "expression": { - "type": "string", - "description": "The math expression" - }, - "window": { - "type": "string", - "description": "A time duration string" - }, "downsampler": { - "type": "string", - "description": "The reducer" + "description": "The reducer", + "type": "string" }, - "upsampler": { - "type": "string", - "description": "The reducer" + "expression": { + "description": "The math expression", + "type": "string" }, "loadedDimensions": { "additionalProperties": true, "type": "object", "x-grafana-type": "data.DataFrame" + }, + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "A time duration string", + "type": "string" } }, - "additionalProperties": false, - "type": "object", "required": [ "expression", "window", @@ -168,7 +170,7 @@ "upsampler", "loadedDimensions" ], - "description": "QueryType = resample" + "type": "object" } } } diff --git a/experimental/query/expr/types_test.go b/experimental/query/expr/types_test.go index 88339030b..bd7b79a55 100644 --- a/experimental/query/expr/types_test.go +++ b/experimental/query/expr/types_test.go @@ -10,7 +10,7 @@ import ( ) func TestQueryTypeDefinitions(t *testing.T) { - builder, err := schema.NewBuilder(t, + builder, err := schema.NewBuilder( schema.BuilderOptions{ BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", DiscriminatorField: "queryType", @@ -62,5 +62,5 @@ func TestQueryTypeDefinitions(t *testing.T) { }) require.NoError(t, err) - _ = builder.Write("types.json") + builder.UpdateSchemaDefinition(t, "types.json") } diff --git a/experimental/query/schema/builder.go b/experimental/query/schema/builder.go index f87fc10dd..c487973d3 100644 --- a/experimental/query/schema/builder.go +++ b/experimental/query/schema/builder.go @@ -29,7 +29,6 @@ type QueryTypeInfo struct { } type QueryTypeBuilder struct { - t *testing.T opts BuilderOptions reflector *jsonschema.Reflector // Needed to use comments defs []query.QueryTypeDefinition @@ -88,7 +87,7 @@ type BuilderOptions struct { Enums []reflect.Type } -func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder, error) { +func NewBuilder(opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder, error) { r := new(jsonschema.Reflector) r.DoNotReference = true if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { @@ -135,7 +134,6 @@ func NewBuilder(t *testing.T, opts BuilderOptions, inputs ...QueryTypeInfo) (*Qu } b := &QueryTypeBuilder{ - t: t, opts: opts, reflector: r, } @@ -183,8 +181,11 @@ func (b *QueryTypeBuilder) enumify(s *jsonschema.Schema) { } } -func (b *QueryTypeBuilder) Write(outfile string) json.RawMessage { - t := b.t +// Update the schema definition file +// When placed in `static/schema/dataquery.json` folder of a plugin distribution, +// it can be used to advertise various query types +// If the spec contents have changed, the test will fail (but still update the output) +func (b *QueryTypeBuilder) UpdateSchemaDefinition(t *testing.T, outfile string) { t.Helper() now := time.Now().UTC() @@ -201,6 +202,8 @@ func (b *QueryTypeBuilder) Write(outfile string) json.RawMessage { } } } + defs.Kind = "QueryTypeDefinitionList" + defs.ApiVersion = "query.grafana.app/v0alpha1" // The updated schemas for _, def := range b.defs { @@ -248,5 +251,4 @@ func (b *QueryTypeBuilder) Write(outfile string) json.RawMessage { err = os.WriteFile(outfile, out, 0644) require.NoError(t, err, "error writing file") } - return out } From 4ae1f0c8c645a1d24db05a52dd76a0c76e91b20c Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 16 Feb 2024 14:30:58 -0800 Subject: [PATCH 11/71] add settings example --- data/utils/jsoniter/jsoniter.go | 15 +- data/utils/jsoniter/jsoniter_test.go | 3 +- experimental/query/common.jsonschema | 97 ----- experimental/query/definition.go | 76 ---- experimental/query/expr/types_test.go | 66 --- experimental/query/schema/builder.go | 254 ------------ experimental/schema/builder.go | 376 ++++++++++++++++++ experimental/{query => }/schema/enums.go | 0 experimental/{query => }/schema/enums_test.go | 0 .../{query/expr => schema/example}/math.go | 2 +- .../{query/expr => schema/example}/reduce.go | 2 +- .../expr => schema/example}/resample.go | 2 +- .../{query/expr => schema/example}/types.go | 10 +- .../{query/expr => schema/example}/types.json | 21 +- experimental/schema/example/types_test.go | 65 +++ experimental/schema/k8s.go | 52 +++ .../{query/common.go => schema/query.go} | 63 ++- experimental/schema/query.schema.json | 82 ++++ experimental/schema/query_parser.go | 57 +++ .../common_test.go => schema/query_test.go} | 14 +- experimental/schema/settings.go | 27 ++ 21 files changed, 753 insertions(+), 531 deletions(-) delete mode 100644 experimental/query/common.jsonschema delete mode 100644 experimental/query/definition.go delete mode 100644 experimental/query/expr/types_test.go delete mode 100644 experimental/query/schema/builder.go create mode 100644 experimental/schema/builder.go rename experimental/{query => }/schema/enums.go (100%) rename experimental/{query => }/schema/enums_test.go (100%) rename experimental/{query/expr => schema/example}/math.go (98%) rename experimental/{query/expr => schema/example}/reduce.go (98%) rename experimental/{query/expr => schema/example}/resample.go (97%) rename experimental/{query/expr => schema/example}/types.go (83%) rename experimental/{query/expr => schema/example}/types.json (90%) create mode 100644 experimental/schema/example/types_test.go create mode 100644 experimental/schema/k8s.go rename experimental/{query/common.go => schema/query.go} (55%) create mode 100644 experimental/schema/query.schema.json create mode 100644 experimental/schema/query_parser.go rename experimental/{query/common_test.go => schema/query_test.go} (65%) create mode 100644 experimental/schema/settings.go diff --git a/data/utils/jsoniter/jsoniter.go b/data/utils/jsoniter/jsoniter.go index 76516a73f..9ba4de1f5 100644 --- a/data/utils/jsoniter/jsoniter.go +++ b/data/utils/jsoniter/jsoniter.go @@ -114,14 +114,17 @@ func (iter *Iterator) Unmarshal(data []byte, v interface{}) error { return ConfigDefault.Unmarshal(data, v) } -func (iter *Iterator) Parse(cfg j.API, reader io.Reader, bufSize int) (*Iterator, error) { - return &Iterator{j.Parse(cfg, reader, bufSize)}, iter.i.Error +func Parse(cfg j.API, reader io.Reader, bufSize int) (*Iterator, error) { + iter := &Iterator{j.Parse(cfg, reader, bufSize)} + return iter, iter.i.Error } -func (iter *Iterator) ParseBytes(cfg j.API, input []byte) (*Iterator, error) { - return &Iterator{j.ParseBytes(cfg, input)}, iter.i.Error +func ParseBytes(cfg j.API, input []byte) (*Iterator, error) { + iter := &Iterator{j.ParseBytes(cfg, input)} + return iter, iter.i.Error } -func (iter *Iterator) ParseString(cfg j.API, input string) (*Iterator, error) { - return &Iterator{j.ParseString(cfg, input)}, iter.i.Error +func ParseString(cfg j.API, input string) (*Iterator, error) { + iter := &Iterator{j.ParseString(cfg, input)} + return iter, iter.i.Error } diff --git a/data/utils/jsoniter/jsoniter_test.go b/data/utils/jsoniter/jsoniter_test.go index 1f9e07359..fb3e751fd 100644 --- a/data/utils/jsoniter/jsoniter_test.go +++ b/data/utils/jsoniter/jsoniter_test.go @@ -36,8 +36,7 @@ func TestRead(t *testing.T) { func TestParse(t *testing.T) { t.Run("should create a new iterator without any error", func(t *testing.T) { - jiter := NewIterator(j.NewIterator(j.ConfigDefault)) - iter, err := jiter.Parse(ConfigDefault, io.NopCloser(strings.NewReader(`{"test":123}`)), 128) + iter, err := Parse(ConfigDefault, io.NopCloser(strings.NewReader(`{"test":123}`)), 128) require.NoError(t, err) require.NotNil(t, iter) }) diff --git a/experimental/query/common.jsonschema b/experimental/query/common.jsonschema deleted file mode 100644 index 617e55dfe..000000000 --- a/experimental/query/common.jsonschema +++ /dev/null @@ -1,97 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/query/common-query-properties", - "properties": { - "refId": { - "type": "string", - "description": "RefID is the unique identifier of the query, set by the frontend call." - }, - "resultAssertions": { - "properties": { - "type": { - "type": "string", - "description": "Type asserts that the frame matches a known type structure." - }, - "typeVersion": { - "items": { - "type": "integer" - }, - "type": "array", - "maxItems": 2, - "minItems": 2, - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." - }, - "maxBytes": { - "type": "integer", - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" - }, - "maxFrames": { - "type": "integer", - "description": "Maximum frame count" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "typeVersion" - ], - "description": "Optionally define expected query result behavior" - }, - "timeRange": { - "properties": { - "from": { - "type": "string", - "description": "From is the start time of the query." - }, - "to": { - "type": "string", - "description": "To is the end time of the query." - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "from", - "to" - ], - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query" - }, - "datasource": { - "properties": { - "type": { - "type": "string", - "description": "The datasource plugin type" - }, - "uid": { - "type": "string", - "description": "Datasource UID" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type", - "uid" - ], - "description": "The datasource" - }, - "queryType": { - "type": "string", - "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." - }, - "maxDataPoints": { - "type": "integer", - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query." - }, - "intervalMs": { - "type": "number", - "description": "Interval is the suggested duration between time points in a time series query." - }, - "hide": { - "type": "boolean", - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNote this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" - } - }, - "additionalProperties": false, - "type": "object" -} \ No newline at end of file diff --git a/experimental/query/definition.go b/experimental/query/definition.go deleted file mode 100644 index 4e770a230..000000000 --- a/experimental/query/definition.go +++ /dev/null @@ -1,76 +0,0 @@ -package query - -import ( - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" -) - -type TypedQueryReader[Q any] interface { - // Get the query parser for a query type - // The version is split from the end of the discriminator field - ReadQuery( - // Properties that have been parsed off the same node - common CommonQueryProperties, - // An iterator with context for the full node (include common values) - iter *jsoniter.Iterator, - ) (Q, error) -} - -// K8s placeholder -// This will serialize to the same byte array, but does not require all the imports -type QueryTypeDefinition struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` - - Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` -} - -// K8s placeholder -// This will serialize to the same byte array, but does not require all the imports -type QueryTypeDefinitionList struct { - Kind string `json:"kind"` // "QueryTypeDefinitionList", - ApiVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", - - ObjectMeta `json:"metadata,omitempty"` - - Items []QueryTypeDefinition `json:"items"` -} - -// K8s compatible -type ObjectMeta struct { - // The name is for k8s and description, but not used in the schema - Name string `json:"name,omitempty"` - // Changes indicate that *something * changed - ResourceVersion string `json:"resourceVersion,omitempty"` - // Timestamp - CreationTimestamp string `json:"creationTimestamp,omitempty"` -} - -type QueryTypeDefinitionSpec struct { - // DiscriminatorField is the field used to link behavior to this specific - // query type. It is typically "queryType", but can be another field if necessary - DiscriminatorField string `json:"discriminatorField,omitempty"` - - // The discriminator value - DiscriminatorValue string `json:"discriminatorValue,omitempty"` - - // Describe whe the query type is for - Description string `json:"description,omitempty"` - - // The JSONSchema definition for the non-common fields - Schema any `json:"schema"` - - // Examples (include a wrapper) ideally a template! - Examples []QueryExample `json:"examples,omitempty"` - - // Changelog defines the changed from the previous version - // All changes in the same version *must* be backwards compatible - // Only notable changes will be shown here, for the full version history see git! - Changelog []string `json:"changelog,omitempty"` -} - -type QueryExample struct { - // Version identifier or empty if only one exists - Name string `json:"name,omitempty"` - - // An example query - Query any `json:"query"` -} diff --git a/experimental/query/expr/types_test.go b/experimental/query/expr/types_test.go deleted file mode 100644 index bd7b79a55..000000000 --- a/experimental/query/expr/types_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package expr - -import ( - "reflect" - "testing" - - "github.com/grafana/grafana-plugin-sdk-go/experimental/query" - "github.com/grafana/grafana-plugin-sdk-go/experimental/query/schema" - "github.com/stretchr/testify/require" -) - -func TestQueryTypeDefinitions(t *testing.T) { - builder, err := schema.NewBuilder( - schema.BuilderOptions{ - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", - DiscriminatorField: "queryType", - CodePath: "./", - // We need to identify the enum fields explicitly :( - // *AND* have the +enum common for this to work - Enums: []reflect.Type{ - reflect.TypeOf(ReducerSum), // pick an example value (not the root) - reflect.TypeOf(ReduceModeDrop), // pick an example value (not the root) - }, - }, - schema.QueryTypeInfo{ - Discriminator: string(QueryTypeMath), - GoType: reflect.TypeOf(&MathQuery{}), - Examples: []query.QueryExample{ - { - Name: "constant addition", - Query: MathQuery{ - Expression: "$A + 10", - }, - }, - { - Name: "math with two queries", - Query: MathQuery{ - Expression: "$A - $B", - }, - }, - }, - }, - schema.QueryTypeInfo{ - Discriminator: string(QueryTypeReduce), - GoType: reflect.TypeOf(&ReduceQuery{}), - Examples: []query.QueryExample{ - { - Name: "get max value", - Query: ReduceQuery{ - Expression: "$A", - Reducer: ReducerMax, - Settings: ReduceSettings{ - Mode: ReduceModeDrop, - }, - }, - }, - }, - }, - schema.QueryTypeInfo{ - Discriminator: string(QueryTypeResample), - GoType: reflect.TypeOf(&ResampleQuery{}), - }) - require.NoError(t, err) - - builder.UpdateSchemaDefinition(t, "types.json") -} diff --git a/experimental/query/schema/builder.go b/experimental/query/schema/builder.go deleted file mode 100644 index c487973d3..000000000 --- a/experimental/query/schema/builder.go +++ /dev/null @@ -1,254 +0,0 @@ -package schema - -import ( - "encoding/json" - "fmt" - "os" - "reflect" - "regexp" - "strings" - "testing" - "time" - - "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana-plugin-sdk-go/experimental/query" - "github.com/invopop/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type QueryTypeInfo struct { - // The management name - Name string - // The discriminator value (requires the field set in ops) - Discriminator string - // Raw GO type used for reflection - GoType reflect.Type - // Add sample queries - Examples []query.QueryExample -} - -type QueryTypeBuilder struct { - opts BuilderOptions - reflector *jsonschema.Reflector // Needed to use comments - defs []query.QueryTypeDefinition -} - -func (b *QueryTypeBuilder) Add(info QueryTypeInfo) error { - schema := b.reflector.ReflectFromType(info.GoType) - if schema == nil { - return fmt.Errorf("missing schema") - } - - b.enumify(schema) - - // Ignored by k8s anyway - schema.Version = "" - schema.ID = "" - schema.Anchor = "" - - name := info.Name - if name == "" { - name = info.Discriminator - if name == "" { - return fmt.Errorf("missing name or discriminator") - } - } - - if info.Discriminator != "" && b.opts.DiscriminatorField == "" { - return fmt.Errorf("missing discriminator field") - } - - b.defs = append(b.defs, query.QueryTypeDefinition{ - ObjectMeta: query.ObjectMeta{ - Name: name, - }, - Spec: query.QueryTypeDefinitionSpec{ - DiscriminatorField: b.opts.DiscriminatorField, - DiscriminatorValue: info.Discriminator, - Schema: schema, - Examples: info.Examples, - }, - }) - return nil -} - -type BuilderOptions struct { - // ex "github.com/invopop/jsonschema" - BasePackage string - - // ex "./" - CodePath string - - // queryType - DiscriminatorField string - - // explicitly define the enumeration fields - Enums []reflect.Type -} - -func NewBuilder(opts BuilderOptions, inputs ...QueryTypeInfo) (*QueryTypeBuilder, error) { - r := new(jsonschema.Reflector) - r.DoNotReference = true - if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { - return nil, err - } - customMapper := map[reflect.Type]*jsonschema.Schema{ - reflect.TypeOf(data.Frame{}): { - Type: "object", - Extras: map[string]any{ - "x-grafana-type": "data.DataFrame", - }, - AdditionalProperties: jsonschema.TrueSchema, - }, - } - r.Mapper = func(t reflect.Type) *jsonschema.Schema { - return customMapper[t] - } - - if len(opts.Enums) > 0 { - fields, err := findEnumFields(opts.BasePackage, opts.CodePath) - if err != nil { - return nil, err - } - for _, etype := range opts.Enums { - for _, f := range fields { - if f.Name == etype.Name() && f.Package == etype.PkgPath() { - enumValueDescriptions := map[string]string{} - s := &jsonschema.Schema{ - Type: "string", - Extras: map[string]any{ - "x-enum-description": enumValueDescriptions, - }, - } - for _, val := range f.Values { - s.Enum = append(s.Enum, val.Value) - if val.Comment != "" { - enumValueDescriptions[val.Value] = val.Comment - } - } - customMapper[etype] = s - } - } - } - } - - b := &QueryTypeBuilder{ - opts: opts, - reflector: r, - } - for _, input := range inputs { - err := b.Add(input) - if err != nil { - return nil, err - } - } - return b, nil -} - -// whitespaceRegex is the regex for consecutive whitespaces. -var whitespaceRegex = regexp.MustCompile(`\s+`) - -func (b *QueryTypeBuilder) enumify(s *jsonschema.Schema) { - if len(s.Enum) > 0 && s.Extras != nil { - extra, ok := s.Extras["x-enum-description"] - if !ok { - return - } - - lookup, ok := extra.(map[string]string) - if !ok { - return - } - - lines := []string{} - if s.Description != "" { - lines = append(lines, s.Description, "\n") - } - lines = append(lines, "Possible enum values:") - for _, v := range s.Enum { - c := lookup[v.(string)] - c = whitespaceRegex.ReplaceAllString(c, " ") - lines = append(lines, fmt.Sprintf(" - `%q` %s", v, c)) - } - - s.Description = strings.Join(lines, "\n") - return - } - - for pair := s.Properties.Oldest(); pair != nil; pair = pair.Next() { - b.enumify(pair.Value) - } -} - -// Update the schema definition file -// When placed in `static/schema/dataquery.json` folder of a plugin distribution, -// it can be used to advertise various query types -// If the spec contents have changed, the test will fail (but still update the output) -func (b *QueryTypeBuilder) UpdateSchemaDefinition(t *testing.T, outfile string) { - t.Helper() - - now := time.Now().UTC() - rv := fmt.Sprintf("%d", now.UnixMilli()) - - defs := query.QueryTypeDefinitionList{} - byName := make(map[string]*query.QueryTypeDefinition) - body, err := os.ReadFile(outfile) - if err == nil { - err = json.Unmarshal(body, &defs) - if err == nil { - for i, def := range defs.Items { - byName[def.ObjectMeta.Name] = &defs.Items[i] - } - } - } - defs.Kind = "QueryTypeDefinitionList" - defs.ApiVersion = "query.grafana.app/v0alpha1" - - // The updated schemas - for _, def := range b.defs { - found, ok := byName[def.ObjectMeta.Name] - if !ok { - defs.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) - - defs.Items = append(defs.Items, def) - } else { - var o1, o2 interface{} - b1, _ := json.Marshal(def.Spec) - b2, _ := json.Marshal(found.Spec) - _ = json.Unmarshal(b1, &o1) - _ = json.Unmarshal(b2, &o2) - if !reflect.DeepEqual(o1, o2) { - found.ObjectMeta.ResourceVersion = rv - found.Spec = def.Spec - } - delete(byName, def.ObjectMeta.Name) - } - } - - if defs.ObjectMeta.ResourceVersion == "" { - defs.ObjectMeta.ResourceVersion = rv - } - - if len(byName) > 0 { - require.FailNow(t, "query type removed, manually update (for now)") - } - - out, err := json.MarshalIndent(defs, "", " ") - require.NoError(t, err) - - update := false - if err == nil { - if !assert.JSONEq(t, string(out), string(body)) { - update = true - } - } else { - update = true - } - if update { - err = os.WriteFile(outfile, out, 0644) - require.NoError(t, err, "error writing file") - } -} diff --git a/experimental/schema/builder.go b/experimental/schema/builder.go new file mode 100644 index 000000000..6d2a86300 --- /dev/null +++ b/experimental/schema/builder.go @@ -0,0 +1,376 @@ +package schema + +import ( + "encoding/json" + "fmt" + "os" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// SchemaBuilder is a helper function that can be used by +// backend build processes to produce static schema definitions +// This is not intended as runtime code, and is not the only way to +// produce a schema (we may also want/need to use typescript as the source) +type SchemaBuilder struct { + opts BuilderOptions + reflector *jsonschema.Reflector // Needed to use comments + query []QueryTypeDefinition + setting []SettingsDefinition +} + +type BuilderOptions struct { + // ex "github.com/invopop/jsonschema" + BasePackage string + + // ex "./" + CodePath string + + // queryType + DiscriminatorField string + + // explicitly define the enumeration fields + Enums []reflect.Type +} + +type QueryTypeInfo struct { + // The management name + Name string + // The discriminator value (requires the field set in ops) + Discriminator string + // Raw GO type used for reflection + GoType reflect.Type + // Add sample queries + Examples []QueryExample +} + +type SettingTypeInfo struct { + // The management name + Name string + // The discriminator value (requires the field set in ops) + Discriminator string + // Raw GO type used for reflection + GoType reflect.Type + // Map[string]string + SecureGoType reflect.Type +} + +func NewSchemaBuilder(opts BuilderOptions) (*SchemaBuilder, error) { + r := new(jsonschema.Reflector) + r.DoNotReference = true + if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { + return nil, err + } + customMapper := map[reflect.Type]*jsonschema.Schema{ + reflect.TypeOf(data.Frame{}): { + Type: "object", + Extras: map[string]any{ + "x-grafana-type": "data.DataFrame", + }, + AdditionalProperties: jsonschema.TrueSchema, + }, + } + r.Mapper = func(t reflect.Type) *jsonschema.Schema { + return customMapper[t] + } + + if len(opts.Enums) > 0 { + fields, err := findEnumFields(opts.BasePackage, opts.CodePath) + if err != nil { + return nil, err + } + for _, etype := range opts.Enums { + for _, f := range fields { + if f.Name == etype.Name() && f.Package == etype.PkgPath() { + enumValueDescriptions := map[string]string{} + s := &jsonschema.Schema{ + Type: "string", + Extras: map[string]any{ + "x-enum-description": enumValueDescriptions, + }, + } + for _, val := range f.Values { + s.Enum = append(s.Enum, val.Value) + if val.Comment != "" { + enumValueDescriptions[val.Value] = val.Comment + } + } + customMapper[etype] = s + } + } + } + } + + return &SchemaBuilder{ + opts: opts, + reflector: r, + }, nil +} + +func (b *SchemaBuilder) AddQueries(inputs ...QueryTypeInfo) error { + for _, info := range inputs { + schema := b.reflector.ReflectFromType(info.GoType) + if schema == nil { + return fmt.Errorf("missing schema") + } + + b.enumify(schema) + + // used by kube-openapi + schema.Version = "https://json-schema.org/draft-04/schema" + schema.ID = "" + schema.Anchor = "" + + name := info.Name + if name == "" { + name = info.Discriminator + if name == "" { + return fmt.Errorf("missing name or discriminator") + } + } + + if info.Discriminator != "" && b.opts.DiscriminatorField == "" { + return fmt.Errorf("missing discriminator field") + } + + b.query = append(b.query, QueryTypeDefinition{ + ObjectMeta: ObjectMeta{ + Name: name, + }, + Spec: QueryTypeDefinitionSpec{ + DiscriminatorField: b.opts.DiscriminatorField, + DiscriminatorValue: info.Discriminator, + QuerySchema: schema, + Examples: info.Examples, + }, + }) + } + return nil +} + +func (b *SchemaBuilder) AddSettings(inputs ...SettingTypeInfo) error { + for _, info := range inputs { + schema := b.reflector.ReflectFromType(info.GoType) + if schema == nil { + return fmt.Errorf("missing schema") + } + + b.enumify(schema) + + // used by kube-openapi + schema.Version = "https://json-schema.org/draft-04/schema" + schema.ID = "" + schema.Anchor = "" + + name := info.Name + if name == "" { + name = info.Discriminator + if name == "" { + return fmt.Errorf("missing name or discriminator") + } + } + + if info.Discriminator != "" && b.opts.DiscriminatorField == "" { + return fmt.Errorf("missing discriminator field") + } + + b.setting = append(b.setting, SettingsDefinition{ + ObjectMeta: ObjectMeta{ + Name: name, + }, + Spec: SettingsDefinitionSpec{ + DiscriminatorField: b.opts.DiscriminatorField, + DiscriminatorValue: info.Discriminator, + JSONDataSchema: schema, + }, + }) + } + return nil +} + +// whitespaceRegex is the regex for consecutive whitespaces. +var whitespaceRegex = regexp.MustCompile(`\s+`) + +func (b *SchemaBuilder) enumify(s *jsonschema.Schema) { + if len(s.Enum) > 0 && s.Extras != nil { + extra, ok := s.Extras["x-enum-description"] + if !ok { + return + } + + lookup, ok := extra.(map[string]string) + if !ok { + return + } + + lines := []string{} + if s.Description != "" { + lines = append(lines, s.Description, "\n") + } + lines = append(lines, "Possible enum values:") + for _, v := range s.Enum { + c := lookup[v.(string)] + c = whitespaceRegex.ReplaceAllString(c, " ") + lines = append(lines, fmt.Sprintf(" - `%q` %s", v, c)) + } + + s.Description = strings.Join(lines, "\n") + return + } + + for pair := s.Properties.Oldest(); pair != nil; pair = pair.Next() { + b.enumify(pair.Value) + } +} + +// Update the schema definition file +// When placed in `static/schema/query.schema.json` folder of a plugin distribution, +// it can be used to advertise various query types +// If the spec contents have changed, the test will fail (but still update the output) +func (b *SchemaBuilder) UpdateQueryDefinition(t *testing.T, outfile string) { + t.Helper() + + now := time.Now().UTC() + rv := fmt.Sprintf("%d", now.UnixMilli()) + + defs := QueryTypeDefinitionList{} + byName := make(map[string]*QueryTypeDefinition) + body, err := os.ReadFile(outfile) + if err == nil { + err = json.Unmarshal(body, &defs) + if err == nil { + for i, def := range defs.Items { + byName[def.ObjectMeta.Name] = &defs.Items[i] + } + } + } + defs.Kind = "QueryTypeDefinitionList" + defs.ApiVersion = "query.grafana.app/v0alpha1" + + // The updated schemas + for _, def := range b.query { + found, ok := byName[def.ObjectMeta.Name] + if !ok { + defs.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) + + defs.Items = append(defs.Items, def) + } else { + var o1, o2 interface{} + b1, _ := json.Marshal(def.Spec) + b2, _ := json.Marshal(found.Spec) + _ = json.Unmarshal(b1, &o1) + _ = json.Unmarshal(b2, &o2) + if !reflect.DeepEqual(o1, o2) { + found.ObjectMeta.ResourceVersion = rv + found.Spec = def.Spec + } + delete(byName, def.ObjectMeta.Name) + } + } + + if defs.ObjectMeta.ResourceVersion == "" { + defs.ObjectMeta.ResourceVersion = rv + } + + if len(byName) > 0 { + require.FailNow(t, "query type removed, manually update (for now)") + } + + out, err := json.MarshalIndent(defs, "", " ") + require.NoError(t, err) + + update := false + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true + } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0644) + require.NoError(t, err, "error writing file") + } +} + +// Update the schema definition file +// When placed in `static/schema/query.schema.json` folder of a plugin distribution, +// it can be used to advertise various query types +// If the spec contents have changed, the test will fail (but still update the output) +func (b *SchemaBuilder) UpdateSettingsDefinition(t *testing.T, outfile string) { + t.Helper() + + now := time.Now().UTC() + rv := fmt.Sprintf("%d", now.UnixMilli()) + + defs := QueryTypeDefinitionList{} + byName := make(map[string]*QueryTypeDefinition) + body, err := os.ReadFile(outfile) + if err == nil { + err = json.Unmarshal(body, &defs) + if err == nil { + for i, def := range defs.Items { + byName[def.ObjectMeta.Name] = &defs.Items[i] + } + } + } + defs.Kind = "SettingsDefinitionList" + defs.ApiVersion = "common.grafana.app/v0alpha1" + + // The updated schemas + for _, def := range b.query { + found, ok := byName[def.ObjectMeta.Name] + if !ok { + defs.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) + + defs.Items = append(defs.Items, def) + } else { + var o1, o2 interface{} + b1, _ := json.Marshal(def.Spec) + b2, _ := json.Marshal(found.Spec) + _ = json.Unmarshal(b1, &o1) + _ = json.Unmarshal(b2, &o2) + if !reflect.DeepEqual(o1, o2) { + found.ObjectMeta.ResourceVersion = rv + found.Spec = def.Spec + } + delete(byName, def.ObjectMeta.Name) + } + } + + if defs.ObjectMeta.ResourceVersion == "" { + defs.ObjectMeta.ResourceVersion = rv + } + + if len(byName) > 0 { + require.FailNow(t, "query type removed, manually update (for now)") + } + + out, err := json.MarshalIndent(defs, "", " ") + require.NoError(t, err) + + update := false + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true + } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0644) + require.NoError(t, err, "error writing file") + } +} diff --git a/experimental/query/schema/enums.go b/experimental/schema/enums.go similarity index 100% rename from experimental/query/schema/enums.go rename to experimental/schema/enums.go diff --git a/experimental/query/schema/enums_test.go b/experimental/schema/enums_test.go similarity index 100% rename from experimental/query/schema/enums_test.go rename to experimental/schema/enums_test.go diff --git a/experimental/query/expr/math.go b/experimental/schema/example/math.go similarity index 98% rename from experimental/query/expr/math.go rename to experimental/schema/example/math.go index 2f27b7ff4..da7a35c91 100644 --- a/experimental/query/expr/math.go +++ b/experimental/schema/example/math.go @@ -1,4 +1,4 @@ -package expr +package example import "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" diff --git a/experimental/query/expr/reduce.go b/experimental/schema/example/reduce.go similarity index 98% rename from experimental/query/expr/reduce.go rename to experimental/schema/example/reduce.go index d58262168..3893cc06c 100644 --- a/experimental/query/expr/reduce.go +++ b/experimental/schema/example/reduce.go @@ -1,4 +1,4 @@ -package expr +package example var _ ExpressionQuery = (*ReduceQuery)(nil) diff --git a/experimental/query/expr/resample.go b/experimental/schema/example/resample.go similarity index 97% rename from experimental/query/expr/resample.go rename to experimental/schema/example/resample.go index 4c099dfbd..77f27b1c6 100644 --- a/experimental/query/expr/resample.go +++ b/experimental/schema/example/resample.go @@ -1,4 +1,4 @@ -package expr +package example import "github.com/grafana/grafana-plugin-sdk-go/data" diff --git a/experimental/query/expr/types.go b/experimental/schema/example/types.go similarity index 83% rename from experimental/query/expr/types.go rename to experimental/schema/example/types.go index ef43a35b6..bea68d38f 100644 --- a/experimental/query/expr/types.go +++ b/experimental/schema/example/types.go @@ -1,4 +1,4 @@ -package expr +package example import ( "embed" @@ -6,7 +6,7 @@ import ( "fmt" "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - "github.com/grafana/grafana-plugin-sdk-go/experimental/query" + "github.com/grafana/grafana-plugin-sdk-go/experimental/schema" ) // Supported expression types @@ -29,7 +29,7 @@ type ExpressionQuery interface { Variables() []string } -var _ query.TypedQueryReader[ExpressionQuery] = (*QueyHandler)(nil) +var _ schema.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) type QueyHandler struct{} @@ -41,9 +41,9 @@ func (*QueyHandler) QueryTypeDefinitionsJSON() (json.RawMessage, error) { } // ReadQuery implements query.TypedQueryHandler. -func (*QueyHandler) ReadQuery( +func (*QueyHandler) ParseQuery( // Properties that have been parsed off the same node - common query.CommonQueryProperties, + common schema.CommonQueryProperties, // An iterator with context for the full node (include common values) iter *jsoniter.Iterator, ) (ExpressionQuery, error) { diff --git a/experimental/query/expr/types.json b/experimental/schema/example/types.json similarity index 90% rename from experimental/query/expr/types.json rename to experimental/schema/example/types.json index 1621655fd..7496498fb 100644 --- a/experimental/query/expr/types.json +++ b/experimental/schema/example/types.json @@ -8,13 +8,14 @@ { "metadata": { "name": "math", - "resourceVersion": "1707873191437", + "resourceVersion": "1708120730495", "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { "discriminatorField": "queryType", "discriminatorValue": "math", - "schema": { + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", "additionalProperties": false, "properties": { "expression": { @@ -35,13 +36,13 @@ "examples": [ { "name": "constant addition", - "query": { + "queryPayload": { "expression": "$A + 10" } }, { "name": "math with two queries", - "query": { + "queryPayload": { "expression": "$A - $B" } } @@ -51,13 +52,14 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1707873191437", + "resourceVersion": "1708120730495", "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { "discriminatorField": "queryType", "discriminatorValue": "reduce", - "schema": { + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", "additionalProperties": false, "properties": { "expression": { @@ -117,7 +119,7 @@ "examples": [ { "name": "get max value", - "query": { + "queryPayload": { "expression": "$A", "reducer": "max", "settings": { @@ -131,13 +133,14 @@ { "metadata": { "name": "resample", - "resourceVersion": "1707873191437", + "resourceVersion": "1708120730495", "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { "discriminatorField": "queryType", "discriminatorValue": "resample", - "schema": { + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", "additionalProperties": false, "description": "QueryType = resample", "properties": { diff --git a/experimental/schema/example/types_test.go b/experimental/schema/example/types_test.go new file mode 100644 index 000000000..26656f141 --- /dev/null +++ b/experimental/schema/example/types_test.go @@ -0,0 +1,65 @@ +package example + +import ( + "reflect" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/schema" + "github.com/stretchr/testify/require" +) + +func TestQueryTypeDefinitions(t *testing.T) { + builder, err := schema.NewSchemaBuilder(schema.BuilderOptions{ + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/schema/example", + DiscriminatorField: "queryType", + CodePath: "./", + // We need to identify the enum fields explicitly :( + // *AND* have the +enum common for this to work + Enums: []reflect.Type{ + reflect.TypeOf(ReducerSum), // pick an example value (not the root) + reflect.TypeOf(ReduceModeDrop), // pick an example value (not the root) + }, + }) + require.NoError(t, err) + err = builder.AddQueries(schema.QueryTypeInfo{ + Discriminator: string(QueryTypeMath), + GoType: reflect.TypeOf(&MathQuery{}), + Examples: []schema.QueryExample{ + { + Name: "constant addition", + QueryPayload: MathQuery{ + Expression: "$A + 10", + }, + }, + { + Name: "math with two queries", + QueryPayload: MathQuery{ + Expression: "$A - $B", + }, + }, + }, + }, + schema.QueryTypeInfo{ + Discriminator: string(QueryTypeReduce), + GoType: reflect.TypeOf(&ReduceQuery{}), + Examples: []schema.QueryExample{ + { + Name: "get max value", + QueryPayload: ReduceQuery{ + Expression: "$A", + Reducer: ReducerMax, + Settings: ReduceSettings{ + Mode: ReduceModeDrop, + }, + }, + }, + }, + }, + schema.QueryTypeInfo{ + Discriminator: string(QueryTypeResample), + GoType: reflect.TypeOf(&ResampleQuery{}), + }) + require.NoError(t, err) + + builder.UpdateQueryDefinition(t, "types.json") +} diff --git a/experimental/schema/k8s.go b/experimental/schema/k8s.go new file mode 100644 index 000000000..87614f3b4 --- /dev/null +++ b/experimental/schema/k8s.go @@ -0,0 +1,52 @@ +package schema + +// ObjectMeta is a struct that aims to "look" like a real kubernetes object when +// written to JSON, however it does not require the pile of dependencies +// This is really an internal helper until we decide which dependencies make sense +// to require within the SDK +type ObjectMeta struct { + // The name is for k8s and description, but not used in the schema + Name string `json:"name,omitempty"` + // Changes indicate that *something * changed + ResourceVersion string `json:"resourceVersion,omitempty"` + // Timestamp + CreationTimestamp string `json:"creationTimestamp,omitempty"` +} + +// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition +type QueryTypeDefinition struct { + ObjectMeta ObjectMeta `json:"metadata,omitempty"` + + Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` +} + +// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types +// For simple data sources, there may be only a single query type, however when multiple types +// exist they must be clearly specified with distinct discriminator field+value pairs +type QueryTypeDefinitionList struct { + Kind string `json:"kind"` // "QueryTypeDefinitionList", + ApiVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", + + ObjectMeta `json:"metadata,omitempty"` + + Items []QueryTypeDefinition `json:"items"` +} + +// SettingsDefinition is a kubernetes shaped object that represents a single query definition +type SettingsDefinition struct { + ObjectMeta ObjectMeta `json:"metadata,omitempty"` + + Spec SettingsDefinitionSpec `json:"spec,omitempty"` +} + +// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types +// For simple data sources, there may be only a single query type, however when multiple types +// exist they must be clearly specified with distinct discriminator field+value pairs +type SettingsDefinitionList struct { + Kind string `json:"kind"` // "SettingsDefinitionList", + ApiVersion string `json:"apiVersion"` // "??.common.grafana.app/v0alpha1", + + ObjectMeta `json:"metadata,omitempty"` + + Items []SettingsDefinition `json:"items"` +} diff --git a/experimental/query/common.go b/experimental/schema/query.go similarity index 55% rename from experimental/query/common.go rename to experimental/schema/query.go index 0d4688814..f8763766f 100644 --- a/experimental/query/common.go +++ b/experimental/schema/query.go @@ -1,4 +1,4 @@ -package query +package schema import ( "embed" @@ -7,6 +7,50 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" ) +type QueryTypeDefinitionSpec struct { + // DiscriminatorField is the field used to link behavior to this specific + // query type. It is typically "queryType", but can be another field if necessary + DiscriminatorField string `json:"discriminatorField,omitempty"` + + // The discriminator value + DiscriminatorValue string `json:"discriminatorValue,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // The query schema represents the properties that can be sent to the API + // In many cases, this may be the same properties that are saved in a dashboard + // In the case where the save model is different, we must also specify a save model + QuerySchema any `json:"querySchema"` + + // The save model defines properties that can be saved into dashboard or similar + // These values are processed by frontend components and then sent to the api + // When specified, this schema will be used to validate saved objects rather than + // the query schema + SaveModel any `json:"saveModel,omitempty"` + + // Examples (include a wrapper) ideally a template! + Examples []QueryExample `json:"examples,omitempty"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} + +type QueryExample struct { + // Version identifier or empty if only one exists + Name string `json:"name,omitempty"` + + // An example payload -- this should not require the frontend code to + // pre-process anything + QueryPayload any `json:"queryPayload,omitempty"` + + // An example save model -- this will require frontend code to convert it + // into a valid query payload + SaveModel any `json:"saveModel,omitempty"` +} + type CommonQueryProperties struct { // RefID is the unique identifier of the query, set by the frontend call. RefID string `json:"refId,omitempty"` @@ -16,6 +60,7 @@ type CommonQueryProperties struct { // TimeRange represents the query range // NOTE: unlike generic /ds/query, we can now send explicit time values in each query + // NOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly TimeRange *TimeRange `json:"timeRange,omitempty"` // The datasource @@ -29,13 +74,17 @@ type CommonQueryProperties struct { QueryType string `json:"queryType,omitempty"` // MaxDataPoints is the maximum number of data points that should be returned from a time series query. + // NOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated + // from the number of pixels visible in a visualization MaxDataPoints int64 `json:"maxDataPoints,omitempty"` // Interval is the suggested duration between time points in a time series query. + // NOTE: the values for intervalMs is not saved in the query model. It is typically calculated + // from the interval required to fill a pixels in the visualization IntervalMS float64 `json:"intervalMs,omitempty"` // true if query is disabled (ie should not be returned to the dashboard) - // Note this does not always imply that the query should not be executed since + // NOTE: this does not always imply that the query should not be executed since // the results from a hidden query may be used as the input to other queries (SSE etc) Hide bool `json:"hide,omitempty"` } @@ -78,15 +127,7 @@ type ResultAssertions struct { MaxFrames int64 `json:"maxFrames,omitempty"` } -// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing -type GenericDataQuery struct { - CommonQueryProperties `json:",inline"` - - // Additional Properties (that live at the root) - Additional map[string]any `json:",inline"` -} - -//go:embed common.jsonschema +//go:embed query.schema.json var f embed.FS // Get the cached feature list (exposed as a k8s resource) diff --git a/experimental/schema/query.schema.json b/experimental/schema/query.schema.json new file mode 100644 index 000000000..57b2ad824 --- /dev/null +++ b/experimental/schema/query.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema", + "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/schema/common-query-properties", + "properties": { + "refId": { + "type": "string" + }, + "resultAssertions": { + "properties": { + "type": { + "type": "string" + }, + "typeVersion": { + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2 + }, + "maxBytes": { + "type": "integer" + }, + "maxFrames": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "typeVersion" + ] + }, + "timeRange": { + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "from", + "to" + ] + }, + "datasource": { + "properties": { + "type": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type", + "uid" + ] + }, + "queryType": { + "type": "string" + }, + "maxDataPoints": { + "type": "integer" + }, + "intervalMs": { + "type": "number" + }, + "hide": { + "type": "boolean" + } + }, + "additionalProperties": false, + "type": "object", + "description": "Query properties shared by all data sources" +} \ No newline at end of file diff --git a/experimental/schema/query_parser.go b/experimental/schema/query_parser.go new file mode 100644 index 000000000..eb72fed0b --- /dev/null +++ b/experimental/schema/query_parser.go @@ -0,0 +1,57 @@ +package schema + +import ( + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" +) + +// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing +type GenericDataQuery struct { + CommonQueryProperties `json:",inline"` + + // Additional Properties (that live at the root) + Additional map[string]any `json:",inline"` +} + +// Generic query parser pattern. +type TypedQueryParser[Q any] interface { + // Get the query parser for a query type + // The version is split from the end of the discriminator field + ParseQuery( + // Properties that have been parsed off the same node + common CommonQueryProperties, + // An iterator with context for the full node (include common values) + iter *jsoniter.Iterator, + ) (Q, error) +} + +var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) + +type GenericQueryParser struct{} + +var commonKeys = map[string]bool{ + "refId": true, + "resultAssertions": true, + "timeRange": true, + "datasource": true, + "datasourceId": true, + "queryType": true, + "maxDataPoints": true, + "intervalMs": true, + "hide": true, +} + +// ParseQuery implements TypedQueryParser. +func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator) (GenericDataQuery, error) { + q := GenericDataQuery{CommonQueryProperties: common, Additional: make(map[string]any)} + field, err := iter.ReadObject() + for field != "" && err == nil { + if !commonKeys[field] { + q.Additional[field], err = iter.Read() + if err != nil { + return q, err + } + } + field, err = iter.ReadObject() + } + return q, err +} diff --git a/experimental/query/common_test.go b/experimental/schema/query_test.go similarity index 65% rename from experimental/query/common_test.go rename to experimental/schema/query_test.go index e53da0b0d..1fa31958d 100644 --- a/experimental/query/common_test.go +++ b/experimental/schema/query_test.go @@ -1,7 +1,8 @@ -package query +package schema import ( "encoding/json" + "fmt" "os" "testing" @@ -17,6 +18,15 @@ func TestCommonSupport(t *testing.T) { require.NoError(t, err) query := r.Reflect(&CommonQueryProperties{}) + query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi + query.Description = "Query properties shared by all data sources" + + // Write the map of values ignored by the common parser + fmt.Printf("var commonKeys = map[string]bool{\n") + for pair := query.Properties.Oldest(); pair != nil; pair = pair.Next() { + fmt.Printf(" \"%s\": true,\n", pair.Key) + } + fmt.Printf("}\n") // // Hide this old property query.Properties.Delete("datasourceId") @@ -24,7 +34,7 @@ func TestCommonSupport(t *testing.T) { require.NoError(t, err) update := false - outfile := "common.jsonschema" + outfile := "query.schema.json" body, err := os.ReadFile(outfile) if err == nil { if !assert.JSONEq(t, string(out), string(body)) { diff --git a/experimental/schema/settings.go b/experimental/schema/settings.go new file mode 100644 index 000000000..72e3a6809 --- /dev/null +++ b/experimental/schema/settings.go @@ -0,0 +1,27 @@ +package schema + +type SettingsDefinitionSpec struct { + // DiscriminatorField is the field used to link behavior to this specific + // query type. It is typically "queryType", but can be another field if necessary + DiscriminatorField string `json:"discriminatorField,omitempty"` + + // The discriminator value + DiscriminatorValue string `json:"discriminatorValue,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // The query schema represents the properties that can be sent to the API + // In many cases, this may be the same properties that are saved in a dashboard + // In the case where the save model is different, we must also specify a save model + JSONDataSchema any `json:"jsonDataSchema"` + + // JSON schema defining the properties needed in secure json + // NOTE these must all be string fields + SecureJSONSchema any `json:"secureJsonSchema"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} From 238f7c9d88169791661fb609506148c81856968c Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 16 Feb 2024 18:21:58 -0800 Subject: [PATCH 12/71] fix build --- experimental/schema/builder.go | 24 ++++++++++++------------ experimental/schema/enums.go | 5 ++--- experimental/schema/enums_test.go | 6 ++++-- experimental/schema/example/math.go | 11 +++++++---- experimental/schema/k8s.go | 4 ++-- experimental/schema/query.go | 5 ++--- experimental/schema/query_test.go | 2 +- experimental/testdata/folder.golden.txt | 4 ++-- 8 files changed, 32 insertions(+), 29 deletions(-) diff --git a/experimental/schema/builder.go b/experimental/schema/builder.go index 6d2a86300..7e9c11bf2 100644 --- a/experimental/schema/builder.go +++ b/experimental/schema/builder.go @@ -20,7 +20,7 @@ import ( // backend build processes to produce static schema definitions // This is not intended as runtime code, and is not the only way to // produce a schema (we may also want/need to use typescript as the source) -type SchemaBuilder struct { +type Builder struct { opts BuilderOptions reflector *jsonschema.Reflector // Needed to use comments query []QueryTypeDefinition @@ -63,7 +63,7 @@ type SettingTypeInfo struct { SecureGoType reflect.Type } -func NewSchemaBuilder(opts BuilderOptions) (*SchemaBuilder, error) { +func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { r := new(jsonschema.Reflector) r.DoNotReference = true if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { @@ -109,13 +109,13 @@ func NewSchemaBuilder(opts BuilderOptions) (*SchemaBuilder, error) { } } - return &SchemaBuilder{ + return &Builder{ opts: opts, reflector: r, }, nil } -func (b *SchemaBuilder) AddQueries(inputs ...QueryTypeInfo) error { +func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { for _, info := range inputs { schema := b.reflector.ReflectFromType(info.GoType) if schema == nil { @@ -156,7 +156,7 @@ func (b *SchemaBuilder) AddQueries(inputs ...QueryTypeInfo) error { return nil } -func (b *SchemaBuilder) AddSettings(inputs ...SettingTypeInfo) error { +func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { for _, info := range inputs { schema := b.reflector.ReflectFromType(info.GoType) if schema == nil { @@ -199,7 +199,7 @@ func (b *SchemaBuilder) AddSettings(inputs ...SettingTypeInfo) error { // whitespaceRegex is the regex for consecutive whitespaces. var whitespaceRegex = regexp.MustCompile(`\s+`) -func (b *SchemaBuilder) enumify(s *jsonschema.Schema) { +func (b *Builder) enumify(s *jsonschema.Schema) { if len(s.Enum) > 0 && s.Extras != nil { extra, ok := s.Extras["x-enum-description"] if !ok { @@ -235,7 +235,7 @@ func (b *SchemaBuilder) enumify(s *jsonschema.Schema) { // When placed in `static/schema/query.schema.json` folder of a plugin distribution, // it can be used to advertise various query types // If the spec contents have changed, the test will fail (but still update the output) -func (b *SchemaBuilder) UpdateQueryDefinition(t *testing.T, outfile string) { +func (b *Builder) UpdateQueryDefinition(t *testing.T, outfile string) { t.Helper() now := time.Now().UTC() @@ -253,7 +253,7 @@ func (b *SchemaBuilder) UpdateQueryDefinition(t *testing.T, outfile string) { } } defs.Kind = "QueryTypeDefinitionList" - defs.ApiVersion = "query.grafana.app/v0alpha1" + defs.APIVersion = "query.grafana.app/v0alpha1" // The updated schemas for _, def := range b.query { @@ -298,7 +298,7 @@ func (b *SchemaBuilder) UpdateQueryDefinition(t *testing.T, outfile string) { update = true } if update { - err = os.WriteFile(outfile, out, 0644) + err = os.WriteFile(outfile, out, 0600) require.NoError(t, err, "error writing file") } } @@ -307,7 +307,7 @@ func (b *SchemaBuilder) UpdateQueryDefinition(t *testing.T, outfile string) { // When placed in `static/schema/query.schema.json` folder of a plugin distribution, // it can be used to advertise various query types // If the spec contents have changed, the test will fail (but still update the output) -func (b *SchemaBuilder) UpdateSettingsDefinition(t *testing.T, outfile string) { +func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) { t.Helper() now := time.Now().UTC() @@ -325,7 +325,7 @@ func (b *SchemaBuilder) UpdateSettingsDefinition(t *testing.T, outfile string) { } } defs.Kind = "SettingsDefinitionList" - defs.ApiVersion = "common.grafana.app/v0alpha1" + defs.APIVersion = "common.grafana.app/v0alpha1" // The updated schemas for _, def := range b.query { @@ -370,7 +370,7 @@ func (b *SchemaBuilder) UpdateSettingsDefinition(t *testing.T, outfile string) { update = true } if update { - err = os.WriteFile(outfile, out, 0644) + err = os.WriteFile(outfile, out, 0600) require.NoError(t, err, "error writing file") } } diff --git a/experimental/schema/enums.go b/experimental/schema/enums.go index 8fa9d4cc9..64ba8544b 100644 --- a/experimental/schema/enums.go +++ b/experimental/schema/enums.go @@ -50,6 +50,7 @@ func findEnumFields(base, path string) ([]EnumField, error) { fields := make([]EnumField, 0) field := &EnumField{} + dp := &doc.Package{} for pkg, p := range dict { for _, f := range p { @@ -67,7 +68,7 @@ func findEnumFields(base, path string) ([]EnumField, error) { txt = gtxt gtxt = "" } - txt = strings.TrimSpace(doc.Synopsis(txt)) + txt = strings.TrimSpace(dp.Synopsis(txt)) if strings.HasSuffix(txt, "+enum") { fields = append(fields, EnumField{ Package: pkg, @@ -75,7 +76,6 @@ func findEnumFields(base, path string) ([]EnumField, error) { Comment: strings.TrimSpace(strings.TrimSuffix(txt, "+enum")), }) field = &fields[len(fields)-1] - //fmt.Printf("ENUM: %s.%s // %s\n", pkg, typ, txt) } } case *ast.ValueSpec: @@ -91,7 +91,6 @@ func findEnumFields(base, path string) ([]EnumField, error) { val := strings.TrimPrefix(v.Value, `"`) val = strings.TrimSuffix(val, `"`) txt = strings.TrimSpace(txt) - //fmt.Printf("%s // %s // %s\n", typ, val, txt) field.Values = append(field.Values, EnumValue{ Value: val, Comment: txt, diff --git a/experimental/schema/enums_test.go b/experimental/schema/enums_test.go index b5b751d3a..ef99a6597 100644 --- a/experimental/schema/enums_test.go +++ b/experimental/schema/enums_test.go @@ -10,11 +10,13 @@ import ( func TestFindEnums(t *testing.T) { fields, err := findEnumFields( - "github.com/grafana/grafana-plugin-sdk-go/experimental/query/expr", - "../expr") + "github.com/grafana/grafana-plugin-sdk-go/experimental/schema", + "./example") require.NoError(t, err) out, err := json.MarshalIndent(fields, "", " ") require.NoError(t, err) fmt.Printf("%s", string(out)) + + require.Equal(t, 3, len(fields)) } diff --git a/experimental/schema/example/math.go b/experimental/schema/example/math.go index da7a35c91..0ef5d43c6 100644 --- a/experimental/schema/example/math.go +++ b/experimental/schema/example/math.go @@ -20,7 +20,9 @@ func (q *MathQuery) Variables() []string { return q.variables } -func readMathQuery(iter *jsoniter.Iterator) (q *MathQuery, err error) { +func readMathQuery(iter *jsoniter.Iterator) (*MathQuery, error) { + var q *MathQuery + var err error fname := "" for fname, err = iter.ReadObject(); fname != "" && err == nil; fname, err = iter.ReadObject() { switch fname { @@ -29,8 +31,9 @@ func readMathQuery(iter *jsoniter.Iterator) (q *MathQuery, err error) { if err != nil { return q, err } - // TODO actually parse the expression - q.Expression = temp + q = &MathQuery{ + Expression: temp, + } default: _, err = iter.ReadAny() // eat up the unused fields @@ -39,5 +42,5 @@ func readMathQuery(iter *jsoniter.Iterator) (q *MathQuery, err error) { } } } - return + return q, nil } diff --git a/experimental/schema/k8s.go b/experimental/schema/k8s.go index 87614f3b4..f049a9ace 100644 --- a/experimental/schema/k8s.go +++ b/experimental/schema/k8s.go @@ -25,7 +25,7 @@ type QueryTypeDefinition struct { // exist they must be clearly specified with distinct discriminator field+value pairs type QueryTypeDefinitionList struct { Kind string `json:"kind"` // "QueryTypeDefinitionList", - ApiVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", + APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", ObjectMeta `json:"metadata,omitempty"` @@ -44,7 +44,7 @@ type SettingsDefinition struct { // exist they must be clearly specified with distinct discriminator field+value pairs type SettingsDefinitionList struct { Kind string `json:"kind"` // "SettingsDefinitionList", - ApiVersion string `json:"apiVersion"` // "??.common.grafana.app/v0alpha1", + APIVersion string `json:"apiVersion"` // "??.common.grafana.app/v0alpha1", ObjectMeta `json:"metadata,omitempty"` diff --git a/experimental/schema/query.go b/experimental/schema/query.go index f8763766f..1764fe250 100644 --- a/experimental/schema/query.go +++ b/experimental/schema/query.go @@ -67,7 +67,7 @@ type CommonQueryProperties struct { Datasource *DataSourceRef `json:"datasource,omitempty"` // Deprecated -- use datasource ref instead - DatasourceId int64 `json:"datasourceId,omitempty"` + DatasourceID int64 `json:"datasourceId,omitempty"` // QueryType is an optional identifier for the type of query. // It can be used to distinguish different types of queries. @@ -96,8 +96,7 @@ type DataSourceRef struct { // Datasource UID UID string `json:"uid"` - // ?? the datasource API version - // ApiVersion string `json:"apiVersion"` + // ?? the datasource API version? (just version, not the group? type | apiVersion?) } // TimeRange represents a time range for a query and is a property of DataQuery. diff --git a/experimental/schema/query_test.go b/experimental/schema/query_test.go index 1fa31958d..b311fe351 100644 --- a/experimental/schema/query_test.go +++ b/experimental/schema/query_test.go @@ -44,7 +44,7 @@ func TestCommonSupport(t *testing.T) { update = true } if update { - err = os.WriteFile(outfile, out, 0644) + err = os.WriteFile(outfile, out, 0600) require.NoError(t, err, "error writing file") } } diff --git a/experimental/testdata/folder.golden.txt b/experimental/testdata/folder.golden.txt index aff653e7d..b6f411a27 100644 --- a/experimental/testdata/folder.golden.txt +++ b/experimental/testdata/folder.golden.txt @@ -9,7 +9,7 @@ Frame[0] { "pathSeparator": "/" } Name: -Dimensions: 2 Fields by 19 Rows +Dimensions: 2 Fields by 20 Rows +------------------+------------------+ | Name: name | Name: media-type | | Labels: | Labels: | @@ -29,4 +29,4 @@ Dimensions: 2 Fields by 19 Rows ====== TEST DATA RESPONSE (arrow base64) ====== -FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAACAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABMAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABQAAAAAAAAAFAAAAAAAAAA+QAAAAAAAABQAQAAAAAAAAAAAAAAAAAAUAEAAAAAAABQAAAAAAAAAKABAAAAAAAAYwAAAAAAAAAAAAAAAgAAABMAAAAAAAAAAAAAAAAAAAATAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA8QAAAPkAAABSRUFETUUubWRhY3Rpb25zYXV0aGNsaWVudGRhdGFzb3VyY2V0ZXN0ZTJlZXJyb3Jzb3VyY2VmZWF0dXJldG9nZ2xlc2ZpbGVpbmZvLmdvZmlsZWluZm9fdGVzdC5nb2ZyYW1lX3NvcnRlci5nb2ZyYW1lX3NvcnRlcl90ZXN0LmdvZ29sZGVuX3Jlc3BvbnNlX2NoZWNrZXIuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlcl90ZXN0LmdvaHR0cF9sb2dnZXJtYWNyb3Ntb2Nrb2F1dGh0b2tlbnJldHJpZXZlcnJlc3RfY2xpZW50LmdvdGVzdGRhdGEAAAAAAAAAAAAAAAAAAAAJAAAAEgAAABsAAAAkAAAALQAAADYAAAA2AAAANgAAADYAAAA2AAAANgAAADYAAAA/AAAASAAAAFEAAABaAAAAWgAAAGMAAABkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnkAAAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAAEAAEAAADYAQAAAAAAAOAAAAAAAAAACAIAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAPgBAABBUlJPVzE= +FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAIAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAA/wAAAAAAAABYAQAAAAAAAAAAAAAAAAAAWAEAAAAAAABUAAAAAAAAALABAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA8QAAAPcAAAD/AAAAAAAAAFJFQURNRS5tZGFjdGlvbnNhdXRoY2xpZW50ZGF0YXNvdXJjZXRlc3RlMmVlcnJvcnNvdXJjZWZlYXR1cmV0b2dnbGVzZmlsZWluZm8uZ29maWxlaW5mb190ZXN0LmdvZnJhbWVfc29ydGVyLmdvZnJhbWVfc29ydGVyX3Rlc3QuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlci5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyX3Rlc3QuZ29odHRwX2xvZ2dlcm1hY3Jvc21vY2tvYXV0aHRva2VucmV0cmlldmVycmVzdF9jbGllbnQuZ29zY2hlbWF0ZXN0ZGF0YQAAAAAAAAAAAAkAAAASAAAAGwAAACQAAAAtAAAANgAAADYAAAA2AAAANgAAADYAAAA2AAAANgAAAD8AAABIAAAAUQAAAFoAAABaAAAAYwAAAGwAAAAAAAAAZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5AAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAAEAAEAAADYAQAAAAAAAOAAAAAAAAAAIAIAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAPgBAABBUlJPVzE= From 1fe487a68394ddf296d160780bb020138aff474f Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 20 Feb 2024 11:08:48 -0500 Subject: [PATCH 13/71] multiple discriminators --- experimental/schema/builder.go | 62 ++++++------- experimental/schema/example/types.json | 102 ++++++++++++---------- experimental/schema/example/types_test.go | 17 ++-- experimental/schema/query.go | 27 +++++- experimental/schema/settings.go | 8 +- 5 files changed, 117 insertions(+), 99 deletions(-) diff --git a/experimental/schema/builder.go b/experimental/schema/builder.go index 7e9c11bf2..edc07845e 100644 --- a/experimental/schema/builder.go +++ b/experimental/schema/builder.go @@ -34,9 +34,6 @@ type BuilderOptions struct { // ex "./" CodePath string - // queryType - DiscriminatorField string - // explicitly define the enumeration fields Enums []reflect.Type } @@ -44,8 +41,8 @@ type BuilderOptions struct { type QueryTypeInfo struct { // The management name Name string - // The discriminator value (requires the field set in ops) - Discriminator string + // Optional discriminators + Discriminators []DiscriminatorFieldValue // Raw GO type used for reflection GoType reflect.Type // Add sample queries @@ -55,8 +52,8 @@ type QueryTypeInfo struct { type SettingTypeInfo struct { // The management name Name string - // The discriminator value (requires the field set in ops) - Discriminator string + // Optional discriminators + Discriminators []DiscriminatorFieldValue // Raw GO type used for reflection GoType reflect.Type // Map[string]string @@ -124,32 +121,33 @@ func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { b.enumify(schema) - // used by kube-openapi - schema.Version = "https://json-schema.org/draft-04/schema" - schema.ID = "" - schema.Anchor = "" - name := info.Name if name == "" { - name = info.Discriminator + for _, dis := range info.Discriminators { + if name != "" { + name += "-" + } + name += dis.Value + } if name == "" { - return fmt.Errorf("missing name or discriminator") + return fmt.Errorf("missing name or discriminators") } } - if info.Discriminator != "" && b.opts.DiscriminatorField == "" { - return fmt.Errorf("missing discriminator field") - } + // We need to be careful to only use draft-04 so that this is possible to use + // with kube-openapi + schema.Version = "https://json-schema.org/draft-04/schema" + schema.ID = "" + schema.Anchor = "" b.query = append(b.query, QueryTypeDefinition{ ObjectMeta: ObjectMeta{ Name: name, }, Spec: QueryTypeDefinitionSpec{ - DiscriminatorField: b.opts.DiscriminatorField, - DiscriminatorValue: info.Discriminator, - QuerySchema: schema, - Examples: info.Examples, + Discriminators: info.Discriminators, + QuerySchema: schema, + Examples: info.Examples, }, }) } @@ -158,6 +156,11 @@ func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { for _, info := range inputs { + name := info.Name + if name == "" { + return fmt.Errorf("missing name") + } + schema := b.reflector.ReflectFromType(info.GoType) if schema == nil { return fmt.Errorf("missing schema") @@ -170,26 +173,13 @@ func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { schema.ID = "" schema.Anchor = "" - name := info.Name - if name == "" { - name = info.Discriminator - if name == "" { - return fmt.Errorf("missing name or discriminator") - } - } - - if info.Discriminator != "" && b.opts.DiscriminatorField == "" { - return fmt.Errorf("missing discriminator field") - } - b.setting = append(b.setting, SettingsDefinition{ ObjectMeta: ObjectMeta{ Name: name, }, Spec: SettingsDefinitionSpec{ - DiscriminatorField: b.opts.DiscriminatorField, - DiscriminatorValue: info.Discriminator, - JSONDataSchema: schema, + Discriminators: info.Discriminators, + JSONDataSchema: schema, }, }) } diff --git a/experimental/schema/example/types.json b/experimental/schema/example/types.json index 7496498fb..fa4874583 100644 --- a/experimental/schema/example/types.json +++ b/experimental/schema/example/types.json @@ -8,30 +8,34 @@ { "metadata": { "name": "math", - "resourceVersion": "1708120730495", + "resourceVersion": "1708445277328", "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { - "discriminatorField": "queryType", - "discriminatorValue": "math", + "discriminators": [ + { + "field": "queryType", + "value": "math" + } + ], "querySchema": { "$schema": "https://json-schema.org/draft-04/schema", - "additionalProperties": false, "properties": { "expression": { + "type": "string", + "minLength": 1, "description": "General math expression", "examples": [ "$A + 1", "$A/$B" - ], - "minLength": 1, - "type": "string" + ] } }, + "additionalProperties": false, + "type": "object", "required": [ "expression" - ], - "type": "object" + ] }, "examples": [ { @@ -52,22 +56,25 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1708120730495", + "resourceVersion": "1708445277328", "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { - "discriminatorField": "queryType", - "discriminatorValue": "reduce", + "discriminators": [ + { + "field": "queryType", + "value": "reduce" + } + ], "querySchema": { "$schema": "https://json-schema.org/draft-04/schema", - "additionalProperties": false, "properties": { "expression": { - "description": "Reference to other query results", - "type": "string" + "type": "string", + "description": "Reference to other query results" }, "reducer": { - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "type": "string", "enum": [ "sum", "mean", @@ -76,45 +83,46 @@ "count", "last" ], - "type": "string", + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", "x-enum-description": { "mean": "The mean", "sum": "The sum" } }, "settings": { - "additionalProperties": false, - "description": "Reducer Options", "properties": { "mode": { - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "type": "string", "enum": [ "dropNN", "replaceNN" ], - "type": "string", + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", "x-enum-description": { "dropNN": "Drop non-numbers", "replaceNN": "Replace non-numbers" } }, "replaceWithValue": { - "description": "Only valid when mode is replace", - "type": "number" + "type": "number", + "description": "Only valid when mode is replace" } }, + "additionalProperties": false, + "type": "object", "required": [ "mode" ], - "type": "object" + "description": "Reducer Options" } }, + "additionalProperties": false, + "type": "object", "required": [ "expression", "reducer", "settings" - ], - "type": "object" + ] }, "examples": [ { @@ -133,39 +141,43 @@ { "metadata": { "name": "resample", - "resourceVersion": "1708120730495", + "resourceVersion": "1708445277328", "creationTimestamp": "2024-02-14T01:12:13Z" }, "spec": { - "discriminatorField": "queryType", - "discriminatorValue": "resample", + "discriminators": [ + { + "field": "queryType", + "value": "resample" + } + ], "querySchema": { "$schema": "https://json-schema.org/draft-04/schema", - "additionalProperties": false, - "description": "QueryType = resample", "properties": { + "expression": { + "type": "string", + "description": "The math expression" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, "downsampler": { - "description": "The reducer", - "type": "string" + "type": "string", + "description": "The reducer" }, - "expression": { - "description": "The math expression", - "type": "string" + "upsampler": { + "type": "string", + "description": "The reducer" }, "loadedDimensions": { "additionalProperties": true, "type": "object", "x-grafana-type": "data.DataFrame" - }, - "upsampler": { - "description": "The reducer", - "type": "string" - }, - "window": { - "description": "A time duration string", - "type": "string" } }, + "additionalProperties": false, + "type": "object", "required": [ "expression", "window", @@ -173,7 +185,7 @@ "upsampler", "loadedDimensions" ], - "type": "object" + "description": "QueryType = resample" } } } diff --git a/experimental/schema/example/types_test.go b/experimental/schema/example/types_test.go index 26656f141..615403471 100644 --- a/experimental/schema/example/types_test.go +++ b/experimental/schema/example/types_test.go @@ -10,9 +10,8 @@ import ( func TestQueryTypeDefinitions(t *testing.T) { builder, err := schema.NewSchemaBuilder(schema.BuilderOptions{ - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/schema/example", - DiscriminatorField: "queryType", - CodePath: "./", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/schema/example", + CodePath: "./", // We need to identify the enum fields explicitly :( // *AND* have the +enum common for this to work Enums: []reflect.Type{ @@ -22,8 +21,8 @@ func TestQueryTypeDefinitions(t *testing.T) { }) require.NoError(t, err) err = builder.AddQueries(schema.QueryTypeInfo{ - Discriminator: string(QueryTypeMath), - GoType: reflect.TypeOf(&MathQuery{}), + Discriminators: schema.NewDiscriminators("queryType", QueryTypeMath), + GoType: reflect.TypeOf(&MathQuery{}), Examples: []schema.QueryExample{ { Name: "constant addition", @@ -40,8 +39,8 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }, schema.QueryTypeInfo{ - Discriminator: string(QueryTypeReduce), - GoType: reflect.TypeOf(&ReduceQuery{}), + Discriminators: schema.NewDiscriminators("queryType", QueryTypeReduce), + GoType: reflect.TypeOf(&ReduceQuery{}), Examples: []schema.QueryExample{ { Name: "get max value", @@ -56,8 +55,8 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }, schema.QueryTypeInfo{ - Discriminator: string(QueryTypeResample), - GoType: reflect.TypeOf(&ResampleQuery{}), + Discriminators: schema.NewDiscriminators("queryType", QueryTypeResample), + GoType: reflect.TypeOf(&ResampleQuery{}), }) require.NoError(t, err) diff --git a/experimental/schema/query.go b/experimental/schema/query.go index 1764fe250..9dc4460bf 100644 --- a/experimental/schema/query.go +++ b/experimental/schema/query.go @@ -3,17 +3,38 @@ package schema import ( "embed" "encoding/json" + "fmt" "github.com/grafana/grafana-plugin-sdk-go/data" ) -type QueryTypeDefinitionSpec struct { +type DiscriminatorFieldValue struct { // DiscriminatorField is the field used to link behavior to this specific // query type. It is typically "queryType", but can be another field if necessary - DiscriminatorField string `json:"discriminatorField,omitempty"` + Field string `json:"field"` // The discriminator value - DiscriminatorValue string `json:"discriminatorValue,omitempty"` + Value string `json:"value"` +} + +// using any since this will often be enumerations +func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { + if len(keyvals)%2 != 0 { + panic("values must be even") + } + dis := []DiscriminatorFieldValue{} + for i := 0; i < len(keyvals); i += 2 { + dis = append(dis, DiscriminatorFieldValue{ + Field: fmt.Sprintf("%v", keyvals[i]), + Value: fmt.Sprintf("%v", keyvals[i+1]), + }) + } + return dis +} + +type QueryTypeDefinitionSpec struct { + // Multiple schemas can be defined using discriminators + Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` // Describe whe the query type is for Description string `json:"description,omitempty"` diff --git a/experimental/schema/settings.go b/experimental/schema/settings.go index 72e3a6809..81db055fd 100644 --- a/experimental/schema/settings.go +++ b/experimental/schema/settings.go @@ -1,12 +1,8 @@ package schema type SettingsDefinitionSpec struct { - // DiscriminatorField is the field used to link behavior to this specific - // query type. It is typically "queryType", but can be another field if necessary - DiscriminatorField string `json:"discriminatorField,omitempty"` - - // The discriminator value - DiscriminatorValue string `json:"discriminatorValue,omitempty"` + // Multiple schemas can be defined using discriminators + Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` // Describe whe the query type is for Description string `json:"description,omitempty"` From f5a35b3a396b7ac55a87d785a527737ccbd691ac Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 21 Feb 2024 16:21:45 -0500 Subject: [PATCH 14/71] difference between save and post --- experimental/schema/builder.go | 149 +++++++++++-- .../schema/example/query.post.schema.json | 199 ++++++++++++++++++ .../schema/example/query.save.schema.json | 172 +++++++++++++++ .../example/{types.json => query.types.json} | 14 +- experimental/schema/example/types.go | 4 +- experimental/schema/example/types_test.go | 2 +- 6 files changed, 514 insertions(+), 26 deletions(-) create mode 100644 experimental/schema/example/query.post.schema.json create mode 100644 experimental/schema/example/query.save.schema.json rename experimental/schema/example/{types.json => query.types.json} (93%) diff --git a/experimental/schema/builder.go b/experimental/schema/builder.go index edc07845e..f45059441 100644 --- a/experimental/schema/builder.go +++ b/experimental/schema/builder.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "reflect" "regexp" "strings" @@ -222,12 +223,13 @@ func (b *Builder) enumify(s *jsonschema.Schema) { } // Update the schema definition file -// When placed in `static/schema/query.schema.json` folder of a plugin distribution, +// When placed in `static/schema/query.types.json` folder of a plugin distribution, // it can be used to advertise various query types // If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateQueryDefinition(t *testing.T, outfile string) { +func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) { t.Helper() + outfile := filepath.Join(outdir, "query.types.json") now := time.Now().UTC() rv := fmt.Sprintf("%d", now.UnixMilli()) @@ -275,22 +277,35 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outfile string) { if len(byName) > 0 { require.FailNow(t, "query type removed, manually update (for now)") } + maybeUpdateFile(t, outfile, defs, body) - out, err := json.MarshalIndent(defs, "", " ") + // Read query info + r := new(jsonschema.Reflector) + r.DoNotReference = true + err = r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/schema", "./") require.NoError(t, err) - update := false - if err == nil { - if !assert.JSONEq(t, string(out), string(body)) { - update = true - } - } else { - update = true - } - if update { - err = os.WriteFile(outfile, out, 0600) - require.NoError(t, err, "error writing file") - } + query := r.Reflect(&CommonQueryProperties{}) + query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi + query.Description = "Query properties shared by all data sources" + + // Now update the query files + //---------------------------- + outfile = filepath.Join(outdir, "query.post.schema.json") + schema, err := toQuerySchema(query, defs, false) + require.NoError(t, err) + + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, schema, body) + + // Now update the query files + //---------------------------- + outfile = filepath.Join(outdir, "query.save.schema.json") + schema, err = toQuerySchema(query, defs, true) + require.NoError(t, err) + + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, schema, body) } // Update the schema definition file @@ -348,7 +363,109 @@ func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) { require.FailNow(t, "query type removed, manually update (for now)") } - out, err := json.MarshalIndent(defs, "", " ") +} + +// This +func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, saveModel bool) (*jsonschema.Schema, error) { + descr := "Query model (the payload sent to /ds/query)" + if saveModel { + descr = "Save model (the payload saved in dashboards and alerts)" + } + + ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true, "timeRange": true} + definitions := make(jsonschema.Definitions) + common := make(map[string]*jsonschema.Schema) + for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { + if saveModel && ignoreForSave[pair.Key] { + continue // + } + definitions[pair.Key] = pair.Value + common[pair.Key] = &jsonschema.Schema{Ref: "#/definitions/" + pair.Key} + } + + // The types for each query type + queryTypes := []*jsonschema.Schema{} + for _, qt := range defs.Items { + node, err := asJSONSchema(qt.Spec.QuerySchema) + node.Version = "" + if err != nil { + return nil, fmt.Errorf("error reading query types schema: %s // %w", qt.ObjectMeta.Name, err) + } + if node == nil { + return nil, fmt.Errorf("missing query schema: %s // %v", qt.ObjectMeta.Name, qt) + } + + // Match all discriminators + for _, d := range qt.Spec.Discriminators { + ds, ok := node.Properties.Get(d.Field) + if !ok { + ds = &jsonschema.Schema{Type: "string"} + node.Properties.Set(d.Field, ds) + } + ds.Pattern = `^` + d.Value + `$` + node.Required = append(node.Required, d.Field) + } + + queryTypes = append(queryTypes, node) + } + + // Single node -- just union the global and local properties + if len(queryTypes) == 1 { + node := queryTypes[0] + node.Version = "https://json-schema.org/draft-04/schema" + node.Description = descr + for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { + _, found := node.Properties.Get(pair.Key) + if found { + continue + } + node.Properties.Set(pair.Key, pair.Value) + } + return node, nil + } + + s := &jsonschema.Schema{ + Type: "object", + Version: "https://json-schema.org/draft-04/schema", + Properties: jsonschema.NewProperties(), + Definitions: make(jsonschema.Definitions), + Description: descr, + } + + for _, qt := range queryTypes { + qt.Required = append(qt.Required, "refId") + + for k, v := range common { + _, found := qt.Properties.Get(k) + if found { + continue + } + qt.Properties.Set(k, v) + } + + s.OneOf = append(s.OneOf, qt) + } + return s, nil +} + +func asJSONSchema(v any) (*jsonschema.Schema, error) { + s, ok := v.(*jsonschema.Schema) + if ok { + return s, nil + } + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + s = &jsonschema.Schema{} + err = json.Unmarshal(b, s) + return s, err +} + +func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { + t.Helper() + + out, err := json.MarshalIndent(value, "", " ") require.NoError(t, err) update := false diff --git a/experimental/schema/example/query.post.schema.json b/experimental/schema/example/query.post.schema.json new file mode 100644 index 000000000..ae7fcb575 --- /dev/null +++ b/experimental/schema/example/query.post.schema.json @@ -0,0 +1,199 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema", + "oneOf": [ + { + "properties": { + "expression": { + "type": "string", + "minLength": 1, + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ] + }, + { + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " + }, + "settings": { + "properties": { + "mode": { + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" + }, + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "mode" + ], + "description": "Reducer Options" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ] + }, + { + "properties": { + "downsampler": { + "type": "string", + "description": "The reducer" + }, + "expression": { + "type": "string", + "description": "The math expression" + }, + "loadedDimensions": { + "additionalProperties": true, + "type": "object" + }, + "upsampler": { + "type": "string", + "description": "The reducer" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "description": "QueryType = resample" + } + ], + "properties": {}, + "type": "object", + "description": "Query model (the payload sent to /ds/query)" +} \ No newline at end of file diff --git a/experimental/schema/example/query.save.schema.json b/experimental/schema/example/query.save.schema.json new file mode 100644 index 000000000..256deaa88 --- /dev/null +++ b/experimental/schema/example/query.save.schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema", + "oneOf": [ + { + "properties": { + "expression": { + "type": "string", + "minLength": 1, + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "datasource": { + "$ref": "#/definitions/datasource" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ] + }, + { + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " + }, + "settings": { + "properties": { + "mode": { + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" + }, + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "mode" + ], + "description": "Reducer Options" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "hide": { + "$ref": "#/definitions/hide" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ] + }, + { + "properties": { + "downsampler": { + "type": "string", + "description": "The reducer" + }, + "expression": { + "type": "string", + "description": "The math expression" + }, + "loadedDimensions": { + "additionalProperties": true, + "type": "object" + }, + "upsampler": { + "type": "string", + "description": "The reducer" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "hide": { + "$ref": "#/definitions/hide" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "description": "QueryType = resample" + } + ], + "properties": {}, + "type": "object", + "description": "Save model (the payload saved in dashboards and alerts)" +} \ No newline at end of file diff --git a/experimental/schema/example/types.json b/experimental/schema/example/query.types.json similarity index 93% rename from experimental/schema/example/types.json rename to experimental/schema/example/query.types.json index fa4874583..cc1cfceb5 100644 --- a/experimental/schema/example/types.json +++ b/experimental/schema/example/query.types.json @@ -2,14 +2,14 @@ "kind": "QueryTypeDefinitionList", "apiVersion": "query.grafana.app/v0alpha1", "metadata": { - "resourceVersion": "1707873133114" + "resourceVersion": "1708548629808" }, "items": [ { "metadata": { "name": "math", - "resourceVersion": "1708445277328", - "creationTimestamp": "2024-02-14T01:12:13Z" + "resourceVersion": "1708548629808", + "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { "discriminators": [ @@ -56,8 +56,8 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1708445277328", - "creationTimestamp": "2024-02-14T01:12:13Z" + "resourceVersion": "1708548629808", + "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { "discriminators": [ @@ -141,8 +141,8 @@ { "metadata": { "name": "resample", - "resourceVersion": "1708445277328", - "creationTimestamp": "2024-02-14T01:12:13Z" + "resourceVersion": "1708548629808", + "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { "discriminators": [ diff --git a/experimental/schema/example/types.go b/experimental/schema/example/types.go index bea68d38f..899bac464 100644 --- a/experimental/schema/example/types.go +++ b/experimental/schema/example/types.go @@ -33,11 +33,11 @@ var _ schema.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) type QueyHandler struct{} -//go:embed types.json +//go:embed query.types.json var f embed.FS func (*QueyHandler) QueryTypeDefinitionsJSON() (json.RawMessage, error) { - return f.ReadFile("types.json") + return f.ReadFile("query.types.json") } // ReadQuery implements query.TypedQueryHandler. diff --git a/experimental/schema/example/types_test.go b/experimental/schema/example/types_test.go index 615403471..5d6a6bbc6 100644 --- a/experimental/schema/example/types_test.go +++ b/experimental/schema/example/types_test.go @@ -60,5 +60,5 @@ func TestQueryTypeDefinitions(t *testing.T) { }) require.NoError(t, err) - builder.UpdateQueryDefinition(t, "types.json") + builder.UpdateQueryDefinition(t, "./") } From dc41e8ca5afc10dd73a5fda04c609bf3b88eb622 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 21 Feb 2024 16:35:07 -0500 Subject: [PATCH 15/71] rename to spec --- experimental/schema/builder.go | 483 ------------------ experimental/schema/enums.go | 112 ---- experimental/schema/enums_test.go | 22 - experimental/schema/example/math.go | 46 -- .../schema/example/query.post.schema.json | 199 -------- .../schema/example/query.save.schema.json | 172 ------- experimental/schema/example/query.types.json | 193 ------- experimental/schema/example/reduce.go | 57 --- experimental/schema/example/resample.go | 28 - experimental/schema/example/types.go | 64 --- experimental/schema/example/types_test.go | 64 --- experimental/schema/k8s.go | 52 -- experimental/schema/query.go | 157 ------ experimental/schema/query.schema.json | 82 --- experimental/schema/query_parser.go | 57 --- experimental/schema/query_test.go | 50 -- experimental/schema/settings.go | 23 - experimental/testdata/folder.golden.txt | 2 +- 18 files changed, 1 insertion(+), 1862 deletions(-) delete mode 100644 experimental/schema/builder.go delete mode 100644 experimental/schema/enums.go delete mode 100644 experimental/schema/enums_test.go delete mode 100644 experimental/schema/example/math.go delete mode 100644 experimental/schema/example/query.post.schema.json delete mode 100644 experimental/schema/example/query.save.schema.json delete mode 100644 experimental/schema/example/query.types.json delete mode 100644 experimental/schema/example/reduce.go delete mode 100644 experimental/schema/example/resample.go delete mode 100644 experimental/schema/example/types.go delete mode 100644 experimental/schema/example/types_test.go delete mode 100644 experimental/schema/k8s.go delete mode 100644 experimental/schema/query.go delete mode 100644 experimental/schema/query.schema.json delete mode 100644 experimental/schema/query_parser.go delete mode 100644 experimental/schema/query_test.go delete mode 100644 experimental/schema/settings.go diff --git a/experimental/schema/builder.go b/experimental/schema/builder.go deleted file mode 100644 index f45059441..000000000 --- a/experimental/schema/builder.go +++ /dev/null @@ -1,483 +0,0 @@ -package schema - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "reflect" - "regexp" - "strings" - "testing" - "time" - - "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/invopop/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// SchemaBuilder is a helper function that can be used by -// backend build processes to produce static schema definitions -// This is not intended as runtime code, and is not the only way to -// produce a schema (we may also want/need to use typescript as the source) -type Builder struct { - opts BuilderOptions - reflector *jsonschema.Reflector // Needed to use comments - query []QueryTypeDefinition - setting []SettingsDefinition -} - -type BuilderOptions struct { - // ex "github.com/invopop/jsonschema" - BasePackage string - - // ex "./" - CodePath string - - // explicitly define the enumeration fields - Enums []reflect.Type -} - -type QueryTypeInfo struct { - // The management name - Name string - // Optional discriminators - Discriminators []DiscriminatorFieldValue - // Raw GO type used for reflection - GoType reflect.Type - // Add sample queries - Examples []QueryExample -} - -type SettingTypeInfo struct { - // The management name - Name string - // Optional discriminators - Discriminators []DiscriminatorFieldValue - // Raw GO type used for reflection - GoType reflect.Type - // Map[string]string - SecureGoType reflect.Type -} - -func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { - r := new(jsonschema.Reflector) - r.DoNotReference = true - if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { - return nil, err - } - customMapper := map[reflect.Type]*jsonschema.Schema{ - reflect.TypeOf(data.Frame{}): { - Type: "object", - Extras: map[string]any{ - "x-grafana-type": "data.DataFrame", - }, - AdditionalProperties: jsonschema.TrueSchema, - }, - } - r.Mapper = func(t reflect.Type) *jsonschema.Schema { - return customMapper[t] - } - - if len(opts.Enums) > 0 { - fields, err := findEnumFields(opts.BasePackage, opts.CodePath) - if err != nil { - return nil, err - } - for _, etype := range opts.Enums { - for _, f := range fields { - if f.Name == etype.Name() && f.Package == etype.PkgPath() { - enumValueDescriptions := map[string]string{} - s := &jsonschema.Schema{ - Type: "string", - Extras: map[string]any{ - "x-enum-description": enumValueDescriptions, - }, - } - for _, val := range f.Values { - s.Enum = append(s.Enum, val.Value) - if val.Comment != "" { - enumValueDescriptions[val.Value] = val.Comment - } - } - customMapper[etype] = s - } - } - } - } - - return &Builder{ - opts: opts, - reflector: r, - }, nil -} - -func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { - for _, info := range inputs { - schema := b.reflector.ReflectFromType(info.GoType) - if schema == nil { - return fmt.Errorf("missing schema") - } - - b.enumify(schema) - - name := info.Name - if name == "" { - for _, dis := range info.Discriminators { - if name != "" { - name += "-" - } - name += dis.Value - } - if name == "" { - return fmt.Errorf("missing name or discriminators") - } - } - - // We need to be careful to only use draft-04 so that this is possible to use - // with kube-openapi - schema.Version = "https://json-schema.org/draft-04/schema" - schema.ID = "" - schema.Anchor = "" - - b.query = append(b.query, QueryTypeDefinition{ - ObjectMeta: ObjectMeta{ - Name: name, - }, - Spec: QueryTypeDefinitionSpec{ - Discriminators: info.Discriminators, - QuerySchema: schema, - Examples: info.Examples, - }, - }) - } - return nil -} - -func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { - for _, info := range inputs { - name := info.Name - if name == "" { - return fmt.Errorf("missing name") - } - - schema := b.reflector.ReflectFromType(info.GoType) - if schema == nil { - return fmt.Errorf("missing schema") - } - - b.enumify(schema) - - // used by kube-openapi - schema.Version = "https://json-schema.org/draft-04/schema" - schema.ID = "" - schema.Anchor = "" - - b.setting = append(b.setting, SettingsDefinition{ - ObjectMeta: ObjectMeta{ - Name: name, - }, - Spec: SettingsDefinitionSpec{ - Discriminators: info.Discriminators, - JSONDataSchema: schema, - }, - }) - } - return nil -} - -// whitespaceRegex is the regex for consecutive whitespaces. -var whitespaceRegex = regexp.MustCompile(`\s+`) - -func (b *Builder) enumify(s *jsonschema.Schema) { - if len(s.Enum) > 0 && s.Extras != nil { - extra, ok := s.Extras["x-enum-description"] - if !ok { - return - } - - lookup, ok := extra.(map[string]string) - if !ok { - return - } - - lines := []string{} - if s.Description != "" { - lines = append(lines, s.Description, "\n") - } - lines = append(lines, "Possible enum values:") - for _, v := range s.Enum { - c := lookup[v.(string)] - c = whitespaceRegex.ReplaceAllString(c, " ") - lines = append(lines, fmt.Sprintf(" - `%q` %s", v, c)) - } - - s.Description = strings.Join(lines, "\n") - return - } - - for pair := s.Properties.Oldest(); pair != nil; pair = pair.Next() { - b.enumify(pair.Value) - } -} - -// Update the schema definition file -// When placed in `static/schema/query.types.json` folder of a plugin distribution, -// it can be used to advertise various query types -// If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) { - t.Helper() - - outfile := filepath.Join(outdir, "query.types.json") - now := time.Now().UTC() - rv := fmt.Sprintf("%d", now.UnixMilli()) - - defs := QueryTypeDefinitionList{} - byName := make(map[string]*QueryTypeDefinition) - body, err := os.ReadFile(outfile) - if err == nil { - err = json.Unmarshal(body, &defs) - if err == nil { - for i, def := range defs.Items { - byName[def.ObjectMeta.Name] = &defs.Items[i] - } - } - } - defs.Kind = "QueryTypeDefinitionList" - defs.APIVersion = "query.grafana.app/v0alpha1" - - // The updated schemas - for _, def := range b.query { - found, ok := byName[def.ObjectMeta.Name] - if !ok { - defs.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) - - defs.Items = append(defs.Items, def) - } else { - var o1, o2 interface{} - b1, _ := json.Marshal(def.Spec) - b2, _ := json.Marshal(found.Spec) - _ = json.Unmarshal(b1, &o1) - _ = json.Unmarshal(b2, &o2) - if !reflect.DeepEqual(o1, o2) { - found.ObjectMeta.ResourceVersion = rv - found.Spec = def.Spec - } - delete(byName, def.ObjectMeta.Name) - } - } - - if defs.ObjectMeta.ResourceVersion == "" { - defs.ObjectMeta.ResourceVersion = rv - } - - if len(byName) > 0 { - require.FailNow(t, "query type removed, manually update (for now)") - } - maybeUpdateFile(t, outfile, defs, body) - - // Read query info - r := new(jsonschema.Reflector) - r.DoNotReference = true - err = r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/schema", "./") - require.NoError(t, err) - - query := r.Reflect(&CommonQueryProperties{}) - query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi - query.Description = "Query properties shared by all data sources" - - // Now update the query files - //---------------------------- - outfile = filepath.Join(outdir, "query.post.schema.json") - schema, err := toQuerySchema(query, defs, false) - require.NoError(t, err) - - body, _ = os.ReadFile(outfile) - maybeUpdateFile(t, outfile, schema, body) - - // Now update the query files - //---------------------------- - outfile = filepath.Join(outdir, "query.save.schema.json") - schema, err = toQuerySchema(query, defs, true) - require.NoError(t, err) - - body, _ = os.ReadFile(outfile) - maybeUpdateFile(t, outfile, schema, body) -} - -// Update the schema definition file -// When placed in `static/schema/query.schema.json` folder of a plugin distribution, -// it can be used to advertise various query types -// If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) { - t.Helper() - - now := time.Now().UTC() - rv := fmt.Sprintf("%d", now.UnixMilli()) - - defs := QueryTypeDefinitionList{} - byName := make(map[string]*QueryTypeDefinition) - body, err := os.ReadFile(outfile) - if err == nil { - err = json.Unmarshal(body, &defs) - if err == nil { - for i, def := range defs.Items { - byName[def.ObjectMeta.Name] = &defs.Items[i] - } - } - } - defs.Kind = "SettingsDefinitionList" - defs.APIVersion = "common.grafana.app/v0alpha1" - - // The updated schemas - for _, def := range b.query { - found, ok := byName[def.ObjectMeta.Name] - if !ok { - defs.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) - - defs.Items = append(defs.Items, def) - } else { - var o1, o2 interface{} - b1, _ := json.Marshal(def.Spec) - b2, _ := json.Marshal(found.Spec) - _ = json.Unmarshal(b1, &o1) - _ = json.Unmarshal(b2, &o2) - if !reflect.DeepEqual(o1, o2) { - found.ObjectMeta.ResourceVersion = rv - found.Spec = def.Spec - } - delete(byName, def.ObjectMeta.Name) - } - } - - if defs.ObjectMeta.ResourceVersion == "" { - defs.ObjectMeta.ResourceVersion = rv - } - - if len(byName) > 0 { - require.FailNow(t, "query type removed, manually update (for now)") - } - -} - -// This -func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, saveModel bool) (*jsonschema.Schema, error) { - descr := "Query model (the payload sent to /ds/query)" - if saveModel { - descr = "Save model (the payload saved in dashboards and alerts)" - } - - ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true, "timeRange": true} - definitions := make(jsonschema.Definitions) - common := make(map[string]*jsonschema.Schema) - for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { - if saveModel && ignoreForSave[pair.Key] { - continue // - } - definitions[pair.Key] = pair.Value - common[pair.Key] = &jsonschema.Schema{Ref: "#/definitions/" + pair.Key} - } - - // The types for each query type - queryTypes := []*jsonschema.Schema{} - for _, qt := range defs.Items { - node, err := asJSONSchema(qt.Spec.QuerySchema) - node.Version = "" - if err != nil { - return nil, fmt.Errorf("error reading query types schema: %s // %w", qt.ObjectMeta.Name, err) - } - if node == nil { - return nil, fmt.Errorf("missing query schema: %s // %v", qt.ObjectMeta.Name, qt) - } - - // Match all discriminators - for _, d := range qt.Spec.Discriminators { - ds, ok := node.Properties.Get(d.Field) - if !ok { - ds = &jsonschema.Schema{Type: "string"} - node.Properties.Set(d.Field, ds) - } - ds.Pattern = `^` + d.Value + `$` - node.Required = append(node.Required, d.Field) - } - - queryTypes = append(queryTypes, node) - } - - // Single node -- just union the global and local properties - if len(queryTypes) == 1 { - node := queryTypes[0] - node.Version = "https://json-schema.org/draft-04/schema" - node.Description = descr - for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { - _, found := node.Properties.Get(pair.Key) - if found { - continue - } - node.Properties.Set(pair.Key, pair.Value) - } - return node, nil - } - - s := &jsonschema.Schema{ - Type: "object", - Version: "https://json-schema.org/draft-04/schema", - Properties: jsonschema.NewProperties(), - Definitions: make(jsonschema.Definitions), - Description: descr, - } - - for _, qt := range queryTypes { - qt.Required = append(qt.Required, "refId") - - for k, v := range common { - _, found := qt.Properties.Get(k) - if found { - continue - } - qt.Properties.Set(k, v) - } - - s.OneOf = append(s.OneOf, qt) - } - return s, nil -} - -func asJSONSchema(v any) (*jsonschema.Schema, error) { - s, ok := v.(*jsonschema.Schema) - if ok { - return s, nil - } - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - s = &jsonschema.Schema{} - err = json.Unmarshal(b, s) - return s, err -} - -func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { - t.Helper() - - out, err := json.MarshalIndent(value, "", " ") - require.NoError(t, err) - - update := false - if err == nil { - if !assert.JSONEq(t, string(out), string(body)) { - update = true - } - } else { - update = true - } - if update { - err = os.WriteFile(outfile, out, 0600) - require.NoError(t, err, "error writing file") - } -} diff --git a/experimental/schema/enums.go b/experimental/schema/enums.go deleted file mode 100644 index 64ba8544b..000000000 --- a/experimental/schema/enums.go +++ /dev/null @@ -1,112 +0,0 @@ -package schema - -import ( - "io/fs" - gopath "path" - "path/filepath" - "strings" - - "go/ast" - "go/doc" - "go/parser" - "go/token" -) - -type EnumValue struct { - Value string - Comment string -} - -type EnumField struct { - Package string - Name string - Comment string - Values []EnumValue -} - -func findEnumFields(base, path string) ([]EnumField, error) { - fset := token.NewFileSet() - dict := make(map[string][]*ast.Package) - err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - d, err := parser.ParseDir(fset, path, nil, parser.ParseComments) - if err != nil { - return err - } - for _, v := range d { - // paths may have multiple packages, like for tests - k := gopath.Join(base, path) - dict[k] = append(dict[k], v) - } - } - return nil - }) - if err != nil { - return nil, err - } - - fields := make([]EnumField, 0) - field := &EnumField{} - dp := &doc.Package{} - - for pkg, p := range dict { - for _, f := range p { - gtxt := "" - typ := "" - ast.Inspect(f, func(n ast.Node) bool { - switch x := n.(type) { - case *ast.TypeSpec: - typ = x.Name.String() - if !ast.IsExported(typ) { - typ = "" - } else { - txt := x.Doc.Text() - if txt == "" && gtxt != "" { - txt = gtxt - gtxt = "" - } - txt = strings.TrimSpace(dp.Synopsis(txt)) - if strings.HasSuffix(txt, "+enum") { - fields = append(fields, EnumField{ - Package: pkg, - Name: typ, - Comment: strings.TrimSpace(strings.TrimSuffix(txt, "+enum")), - }) - field = &fields[len(fields)-1] - } - } - case *ast.ValueSpec: - txt := x.Doc.Text() - if txt == "" { - txt = x.Comment.Text() - } - if typ == field.Name { - for _, n := range x.Names { - if ast.IsExported(n.String()) { - v, ok := x.Values[0].(*ast.BasicLit) - if ok { - val := strings.TrimPrefix(v.Value, `"`) - val = strings.TrimSuffix(val, `"`) - txt = strings.TrimSpace(txt) - field.Values = append(field.Values, EnumValue{ - Value: val, - Comment: txt, - }) - } - } - } - } - case *ast.GenDecl: - // remember for the next type - gtxt = x.Doc.Text() - } - return true - }) - } - } - - return fields, nil -} diff --git a/experimental/schema/enums_test.go b/experimental/schema/enums_test.go deleted file mode 100644 index ef99a6597..000000000 --- a/experimental/schema/enums_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package schema - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestFindEnums(t *testing.T) { - fields, err := findEnumFields( - "github.com/grafana/grafana-plugin-sdk-go/experimental/schema", - "./example") - require.NoError(t, err) - - out, err := json.MarshalIndent(fields, "", " ") - require.NoError(t, err) - fmt.Printf("%s", string(out)) - - require.Equal(t, 3, len(fields)) -} diff --git a/experimental/schema/example/math.go b/experimental/schema/example/math.go deleted file mode 100644 index 0ef5d43c6..000000000 --- a/experimental/schema/example/math.go +++ /dev/null @@ -1,46 +0,0 @@ -package example - -import "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - -var _ ExpressionQuery = (*MathQuery)(nil) - -type MathQuery struct { - // General math expression - Expression string `json:"expression" jsonschema:"minLength=1,example=$A + 1,example=$A/$B"` - - // Parsed from the expression - variables []string `json:"-"` -} - -func (*MathQuery) ExpressionQueryType() QueryType { - return QueryTypeMath -} - -func (q *MathQuery) Variables() []string { - return q.variables -} - -func readMathQuery(iter *jsoniter.Iterator) (*MathQuery, error) { - var q *MathQuery - var err error - fname := "" - for fname, err = iter.ReadObject(); fname != "" && err == nil; fname, err = iter.ReadObject() { - switch fname { - case "expression": - temp, err := iter.ReadString() - if err != nil { - return q, err - } - q = &MathQuery{ - Expression: temp, - } - - default: - _, err = iter.ReadAny() // eat up the unused fields - if err != nil { - return nil, err - } - } - } - return q, nil -} diff --git a/experimental/schema/example/query.post.schema.json b/experimental/schema/example/query.post.schema.json deleted file mode 100644 index ae7fcb575..000000000 --- a/experimental/schema/example/query.post.schema.json +++ /dev/null @@ -1,199 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-04/schema", - "oneOf": [ - { - "properties": { - "expression": { - "type": "string", - "minLength": 1, - "description": "General math expression", - "examples": [ - "$A + 1", - "$A/$B" - ] - }, - "queryType": { - "type": "string", - "pattern": "^math$" - }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "queryType", - "refId" - ] - }, - { - "properties": { - "expression": { - "type": "string", - "description": "Reference to other query results" - }, - "reducer": { - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " - }, - "settings": { - "properties": { - "mode": { - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" - }, - "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "mode" - ], - "description": "Reducer Options" - }, - "queryType": { - "type": "string", - "pattern": "^reduce$" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "reducer", - "settings", - "queryType", - "refId" - ] - }, - { - "properties": { - "downsampler": { - "type": "string", - "description": "The reducer" - }, - "expression": { - "type": "string", - "description": "The math expression" - }, - "loadedDimensions": { - "additionalProperties": true, - "type": "object" - }, - "upsampler": { - "type": "string", - "description": "The reducer" - }, - "window": { - "type": "string", - "description": "A time duration string" - }, - "queryType": { - "type": "string", - "pattern": "^resample$" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions", - "queryType", - "refId" - ], - "description": "QueryType = resample" - } - ], - "properties": {}, - "type": "object", - "description": "Query model (the payload sent to /ds/query)" -} \ No newline at end of file diff --git a/experimental/schema/example/query.save.schema.json b/experimental/schema/example/query.save.schema.json deleted file mode 100644 index 256deaa88..000000000 --- a/experimental/schema/example/query.save.schema.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-04/schema", - "oneOf": [ - { - "properties": { - "expression": { - "type": "string", - "minLength": 1, - "description": "General math expression", - "examples": [ - "$A + 1", - "$A/$B" - ] - }, - "queryType": { - "type": "string", - "pattern": "^math$" - }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "datasource": { - "$ref": "#/definitions/datasource" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "queryType", - "refId" - ] - }, - { - "properties": { - "expression": { - "type": "string", - "description": "Reference to other query results" - }, - "reducer": { - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " - }, - "settings": { - "properties": { - "mode": { - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" - }, - "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "mode" - ], - "description": "Reducer Options" - }, - "queryType": { - "type": "string", - "pattern": "^reduce$" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "datasource": { - "$ref": "#/definitions/datasource" - }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, - "hide": { - "$ref": "#/definitions/hide" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "reducer", - "settings", - "queryType", - "refId" - ] - }, - { - "properties": { - "downsampler": { - "type": "string", - "description": "The reducer" - }, - "expression": { - "type": "string", - "description": "The math expression" - }, - "loadedDimensions": { - "additionalProperties": true, - "type": "object" - }, - "upsampler": { - "type": "string", - "description": "The reducer" - }, - "window": { - "type": "string", - "description": "A time duration string" - }, - "queryType": { - "type": "string", - "pattern": "^resample$" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "datasource": { - "$ref": "#/definitions/datasource" - }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, - "hide": { - "$ref": "#/definitions/hide" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions", - "queryType", - "refId" - ], - "description": "QueryType = resample" - } - ], - "properties": {}, - "type": "object", - "description": "Save model (the payload saved in dashboards and alerts)" -} \ No newline at end of file diff --git a/experimental/schema/example/query.types.json b/experimental/schema/example/query.types.json deleted file mode 100644 index cc1cfceb5..000000000 --- a/experimental/schema/example/query.types.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "kind": "QueryTypeDefinitionList", - "apiVersion": "query.grafana.app/v0alpha1", - "metadata": { - "resourceVersion": "1708548629808" - }, - "items": [ - { - "metadata": { - "name": "math", - "resourceVersion": "1708548629808", - "creationTimestamp": "2024-02-21T20:50:29Z" - }, - "spec": { - "discriminators": [ - { - "field": "queryType", - "value": "math" - } - ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", - "properties": { - "expression": { - "type": "string", - "minLength": 1, - "description": "General math expression", - "examples": [ - "$A + 1", - "$A/$B" - ] - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression" - ] - }, - "examples": [ - { - "name": "constant addition", - "queryPayload": { - "expression": "$A + 10" - } - }, - { - "name": "math with two queries", - "queryPayload": { - "expression": "$A - $B" - } - } - ] - } - }, - { - "metadata": { - "name": "reduce", - "resourceVersion": "1708548629808", - "creationTimestamp": "2024-02-21T20:50:29Z" - }, - "spec": { - "discriminators": [ - { - "field": "queryType", - "value": "reduce" - } - ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", - "properties": { - "expression": { - "type": "string", - "description": "Reference to other query results" - }, - "reducer": { - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", - "x-enum-description": { - "mean": "The mean", - "sum": "The sum" - } - }, - "settings": { - "properties": { - "mode": { - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "x-enum-description": { - "dropNN": "Drop non-numbers", - "replaceNN": "Replace non-numbers" - } - }, - "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "mode" - ], - "description": "Reducer Options" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "reducer", - "settings" - ] - }, - "examples": [ - { - "name": "get max value", - "queryPayload": { - "expression": "$A", - "reducer": "max", - "settings": { - "mode": "dropNN" - } - } - } - ] - } - }, - { - "metadata": { - "name": "resample", - "resourceVersion": "1708548629808", - "creationTimestamp": "2024-02-21T20:50:29Z" - }, - "spec": { - "discriminators": [ - { - "field": "queryType", - "value": "resample" - } - ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", - "properties": { - "expression": { - "type": "string", - "description": "The math expression" - }, - "window": { - "type": "string", - "description": "A time duration string" - }, - "downsampler": { - "type": "string", - "description": "The reducer" - }, - "upsampler": { - "type": "string", - "description": "The reducer" - }, - "loadedDimensions": { - "additionalProperties": true, - "type": "object", - "x-grafana-type": "data.DataFrame" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions" - ], - "description": "QueryType = resample" - } - } - } - ] -} \ No newline at end of file diff --git a/experimental/schema/example/reduce.go b/experimental/schema/example/reduce.go deleted file mode 100644 index 3893cc06c..000000000 --- a/experimental/schema/example/reduce.go +++ /dev/null @@ -1,57 +0,0 @@ -package example - -var _ ExpressionQuery = (*ReduceQuery)(nil) - -type ReduceQuery struct { - // Reference to other query results - Expression string `json:"expression"` - - // The reducer - Reducer ReducerID `json:"reducer"` - - // Reducer Options - Settings ReduceSettings `json:"settings"` -} - -func (*ReduceQuery) ExpressionQueryType() QueryType { - return QueryTypeReduce -} - -func (q *ReduceQuery) Variables() []string { - return []string{q.Expression} -} - -type ReduceSettings struct { - // Non-number reduce behavior - Mode ReduceMode `json:"mode"` - - // Only valid when mode is replace - ReplaceWithValue *float64 `json:"replaceWithValue,omitempty"` -} - -// The reducer function -// +enum -type ReducerID string - -const ( - // The sum - ReducerSum ReducerID = "sum" - // The mean - ReducerMean ReducerID = "mean" - ReducerMin ReducerID = "min" - ReducerMax ReducerID = "max" - ReducerCount ReducerID = "count" - ReducerLast ReducerID = "last" -) - -// Non-Number behavior mode -// +enum -type ReduceMode string - -const ( - // Drop non-numbers - ReduceModeDrop ReduceMode = "dropNN" - - // Replace non-numbers - ReduceModeReplace ReduceMode = "replaceNN" -) diff --git a/experimental/schema/example/resample.go b/experimental/schema/example/resample.go deleted file mode 100644 index 77f27b1c6..000000000 --- a/experimental/schema/example/resample.go +++ /dev/null @@ -1,28 +0,0 @@ -package example - -import "github.com/grafana/grafana-plugin-sdk-go/data" - -// QueryType = resample -type ResampleQuery struct { - // The math expression - Expression string `json:"expression"` - - // A time duration string - Window string `json:"window"` - - // The reducer - Downsampler string `json:"downsampler"` - - // The reducer - Upsampler string `json:"upsampler"` - - LoadedDimensions *data.Frame `json:"loadedDimensions"` -} - -func (*ResampleQuery) ExpressionQueryType() QueryType { - return QueryTypeReduce -} - -func (q *ResampleQuery) Variables() []string { - return []string{q.Expression} -} diff --git a/experimental/schema/example/types.go b/experimental/schema/example/types.go deleted file mode 100644 index 899bac464..000000000 --- a/experimental/schema/example/types.go +++ /dev/null @@ -1,64 +0,0 @@ -package example - -import ( - "embed" - "encoding/json" - "fmt" - - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - "github.com/grafana/grafana-plugin-sdk-go/experimental/schema" -) - -// Supported expression types -// +enum -type QueryType string - -const ( - // Math query type - QueryTypeMath QueryType = "math" - - // Reduce query type - QueryTypeReduce QueryType = "reduce" - - // Reduce query type - QueryTypeResample QueryType = "resample" -) - -type ExpressionQuery interface { - ExpressionQueryType() QueryType - Variables() []string -} - -var _ schema.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) - -type QueyHandler struct{} - -//go:embed query.types.json -var f embed.FS - -func (*QueyHandler) QueryTypeDefinitionsJSON() (json.RawMessage, error) { - return f.ReadFile("query.types.json") -} - -// ReadQuery implements query.TypedQueryHandler. -func (*QueyHandler) ParseQuery( - // Properties that have been parsed off the same node - common schema.CommonQueryProperties, - // An iterator with context for the full node (include common values) - iter *jsoniter.Iterator, -) (ExpressionQuery, error) { - qt := QueryType(common.QueryType) - switch qt { - case QueryTypeMath: - return readMathQuery(iter) - - case QueryTypeReduce: - q := &ReduceQuery{} - err := iter.ReadVal(q) - return q, err - - case QueryTypeResample: - return nil, nil - } - return nil, fmt.Errorf("unknown query type") -} diff --git a/experimental/schema/example/types_test.go b/experimental/schema/example/types_test.go deleted file mode 100644 index 5d6a6bbc6..000000000 --- a/experimental/schema/example/types_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package example - -import ( - "reflect" - "testing" - - "github.com/grafana/grafana-plugin-sdk-go/experimental/schema" - "github.com/stretchr/testify/require" -) - -func TestQueryTypeDefinitions(t *testing.T) { - builder, err := schema.NewSchemaBuilder(schema.BuilderOptions{ - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/schema/example", - CodePath: "./", - // We need to identify the enum fields explicitly :( - // *AND* have the +enum common for this to work - Enums: []reflect.Type{ - reflect.TypeOf(ReducerSum), // pick an example value (not the root) - reflect.TypeOf(ReduceModeDrop), // pick an example value (not the root) - }, - }) - require.NoError(t, err) - err = builder.AddQueries(schema.QueryTypeInfo{ - Discriminators: schema.NewDiscriminators("queryType", QueryTypeMath), - GoType: reflect.TypeOf(&MathQuery{}), - Examples: []schema.QueryExample{ - { - Name: "constant addition", - QueryPayload: MathQuery{ - Expression: "$A + 10", - }, - }, - { - Name: "math with two queries", - QueryPayload: MathQuery{ - Expression: "$A - $B", - }, - }, - }, - }, - schema.QueryTypeInfo{ - Discriminators: schema.NewDiscriminators("queryType", QueryTypeReduce), - GoType: reflect.TypeOf(&ReduceQuery{}), - Examples: []schema.QueryExample{ - { - Name: "get max value", - QueryPayload: ReduceQuery{ - Expression: "$A", - Reducer: ReducerMax, - Settings: ReduceSettings{ - Mode: ReduceModeDrop, - }, - }, - }, - }, - }, - schema.QueryTypeInfo{ - Discriminators: schema.NewDiscriminators("queryType", QueryTypeResample), - GoType: reflect.TypeOf(&ResampleQuery{}), - }) - require.NoError(t, err) - - builder.UpdateQueryDefinition(t, "./") -} diff --git a/experimental/schema/k8s.go b/experimental/schema/k8s.go deleted file mode 100644 index f049a9ace..000000000 --- a/experimental/schema/k8s.go +++ /dev/null @@ -1,52 +0,0 @@ -package schema - -// ObjectMeta is a struct that aims to "look" like a real kubernetes object when -// written to JSON, however it does not require the pile of dependencies -// This is really an internal helper until we decide which dependencies make sense -// to require within the SDK -type ObjectMeta struct { - // The name is for k8s and description, but not used in the schema - Name string `json:"name,omitempty"` - // Changes indicate that *something * changed - ResourceVersion string `json:"resourceVersion,omitempty"` - // Timestamp - CreationTimestamp string `json:"creationTimestamp,omitempty"` -} - -// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition -type QueryTypeDefinition struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` - - Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` -} - -// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types -// For simple data sources, there may be only a single query type, however when multiple types -// exist they must be clearly specified with distinct discriminator field+value pairs -type QueryTypeDefinitionList struct { - Kind string `json:"kind"` // "QueryTypeDefinitionList", - APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", - - ObjectMeta `json:"metadata,omitempty"` - - Items []QueryTypeDefinition `json:"items"` -} - -// SettingsDefinition is a kubernetes shaped object that represents a single query definition -type SettingsDefinition struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` - - Spec SettingsDefinitionSpec `json:"spec,omitempty"` -} - -// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types -// For simple data sources, there may be only a single query type, however when multiple types -// exist they must be clearly specified with distinct discriminator field+value pairs -type SettingsDefinitionList struct { - Kind string `json:"kind"` // "SettingsDefinitionList", - APIVersion string `json:"apiVersion"` // "??.common.grafana.app/v0alpha1", - - ObjectMeta `json:"metadata,omitempty"` - - Items []SettingsDefinition `json:"items"` -} diff --git a/experimental/schema/query.go b/experimental/schema/query.go deleted file mode 100644 index 9dc4460bf..000000000 --- a/experimental/schema/query.go +++ /dev/null @@ -1,157 +0,0 @@ -package schema - -import ( - "embed" - "encoding/json" - "fmt" - - "github.com/grafana/grafana-plugin-sdk-go/data" -) - -type DiscriminatorFieldValue struct { - // DiscriminatorField is the field used to link behavior to this specific - // query type. It is typically "queryType", but can be another field if necessary - Field string `json:"field"` - - // The discriminator value - Value string `json:"value"` -} - -// using any since this will often be enumerations -func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { - if len(keyvals)%2 != 0 { - panic("values must be even") - } - dis := []DiscriminatorFieldValue{} - for i := 0; i < len(keyvals); i += 2 { - dis = append(dis, DiscriminatorFieldValue{ - Field: fmt.Sprintf("%v", keyvals[i]), - Value: fmt.Sprintf("%v", keyvals[i+1]), - }) - } - return dis -} - -type QueryTypeDefinitionSpec struct { - // Multiple schemas can be defined using discriminators - Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` - - // Describe whe the query type is for - Description string `json:"description,omitempty"` - - // The query schema represents the properties that can be sent to the API - // In many cases, this may be the same properties that are saved in a dashboard - // In the case where the save model is different, we must also specify a save model - QuerySchema any `json:"querySchema"` - - // The save model defines properties that can be saved into dashboard or similar - // These values are processed by frontend components and then sent to the api - // When specified, this schema will be used to validate saved objects rather than - // the query schema - SaveModel any `json:"saveModel,omitempty"` - - // Examples (include a wrapper) ideally a template! - Examples []QueryExample `json:"examples,omitempty"` - - // Changelog defines the changed from the previous version - // All changes in the same version *must* be backwards compatible - // Only notable changes will be shown here, for the full version history see git! - Changelog []string `json:"changelog,omitempty"` -} - -type QueryExample struct { - // Version identifier or empty if only one exists - Name string `json:"name,omitempty"` - - // An example payload -- this should not require the frontend code to - // pre-process anything - QueryPayload any `json:"queryPayload,omitempty"` - - // An example save model -- this will require frontend code to convert it - // into a valid query payload - SaveModel any `json:"saveModel,omitempty"` -} - -type CommonQueryProperties struct { - // RefID is the unique identifier of the query, set by the frontend call. - RefID string `json:"refId,omitempty"` - - // Optionally define expected query result behavior - ResultAssertions *ResultAssertions `json:"resultAssertions,omitempty"` - - // TimeRange represents the query range - // NOTE: unlike generic /ds/query, we can now send explicit time values in each query - // NOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly - TimeRange *TimeRange `json:"timeRange,omitempty"` - - // The datasource - Datasource *DataSourceRef `json:"datasource,omitempty"` - - // Deprecated -- use datasource ref instead - DatasourceID int64 `json:"datasourceId,omitempty"` - - // QueryType is an optional identifier for the type of query. - // It can be used to distinguish different types of queries. - QueryType string `json:"queryType,omitempty"` - - // MaxDataPoints is the maximum number of data points that should be returned from a time series query. - // NOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated - // from the number of pixels visible in a visualization - MaxDataPoints int64 `json:"maxDataPoints,omitempty"` - - // Interval is the suggested duration between time points in a time series query. - // NOTE: the values for intervalMs is not saved in the query model. It is typically calculated - // from the interval required to fill a pixels in the visualization - IntervalMS float64 `json:"intervalMs,omitempty"` - - // true if query is disabled (ie should not be returned to the dashboard) - // NOTE: this does not always imply that the query should not be executed since - // the results from a hidden query may be used as the input to other queries (SSE etc) - Hide bool `json:"hide,omitempty"` -} - -type DataSourceRef struct { - // The datasource plugin type - Type string `json:"type"` - - // Datasource UID - UID string `json:"uid"` - - // ?? the datasource API version? (just version, not the group? type | apiVersion?) -} - -// TimeRange represents a time range for a query and is a property of DataQuery. -type TimeRange struct { - // From is the start time of the query. - From string `json:"from"` - - // To is the end time of the query. - To string `json:"to"` -} - -// ResultAssertions define the expected response shape and query behavior. This is useful to -// enforce behavior over time. The assertions are passed to the query engine and can be used -// to fail queries *before* returning them to a client (select * from bigquery!) -type ResultAssertions struct { - // Type asserts that the frame matches a known type structure. - Type data.FrameType `json:"type,omitempty"` - - // TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane - // contract documentation https://grafana.github.io/dataplane/contract/. - TypeVersion data.FrameTypeVersion `json:"typeVersion"` - - // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast - MaxBytes int64 `json:"maxBytes,omitempty"` - - // Maximum frame count - MaxFrames int64 `json:"maxFrames,omitempty"` -} - -//go:embed query.schema.json -var f embed.FS - -// Get the cached feature list (exposed as a k8s resource) -func GetCommonJSONSchema() json.RawMessage { - body, _ := f.ReadFile("common.jsonschema") - return body -} diff --git a/experimental/schema/query.schema.json b/experimental/schema/query.schema.json deleted file mode 100644 index 57b2ad824..000000000 --- a/experimental/schema/query.schema.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-04/schema", - "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/schema/common-query-properties", - "properties": { - "refId": { - "type": "string" - }, - "resultAssertions": { - "properties": { - "type": { - "type": "string" - }, - "typeVersion": { - "items": { - "type": "integer" - }, - "type": "array", - "maxItems": 2, - "minItems": 2 - }, - "maxBytes": { - "type": "integer" - }, - "maxFrames": { - "type": "integer" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "typeVersion" - ] - }, - "timeRange": { - "properties": { - "from": { - "type": "string" - }, - "to": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "from", - "to" - ] - }, - "datasource": { - "properties": { - "type": { - "type": "string" - }, - "uid": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type", - "uid" - ] - }, - "queryType": { - "type": "string" - }, - "maxDataPoints": { - "type": "integer" - }, - "intervalMs": { - "type": "number" - }, - "hide": { - "type": "boolean" - } - }, - "additionalProperties": false, - "type": "object", - "description": "Query properties shared by all data sources" -} \ No newline at end of file diff --git a/experimental/schema/query_parser.go b/experimental/schema/query_parser.go deleted file mode 100644 index eb72fed0b..000000000 --- a/experimental/schema/query_parser.go +++ /dev/null @@ -1,57 +0,0 @@ -package schema - -import ( - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" -) - -// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing -type GenericDataQuery struct { - CommonQueryProperties `json:",inline"` - - // Additional Properties (that live at the root) - Additional map[string]any `json:",inline"` -} - -// Generic query parser pattern. -type TypedQueryParser[Q any] interface { - // Get the query parser for a query type - // The version is split from the end of the discriminator field - ParseQuery( - // Properties that have been parsed off the same node - common CommonQueryProperties, - // An iterator with context for the full node (include common values) - iter *jsoniter.Iterator, - ) (Q, error) -} - -var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) - -type GenericQueryParser struct{} - -var commonKeys = map[string]bool{ - "refId": true, - "resultAssertions": true, - "timeRange": true, - "datasource": true, - "datasourceId": true, - "queryType": true, - "maxDataPoints": true, - "intervalMs": true, - "hide": true, -} - -// ParseQuery implements TypedQueryParser. -func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator) (GenericDataQuery, error) { - q := GenericDataQuery{CommonQueryProperties: common, Additional: make(map[string]any)} - field, err := iter.ReadObject() - for field != "" && err == nil { - if !commonKeys[field] { - q.Additional[field], err = iter.Read() - if err != nil { - return q, err - } - } - field, err = iter.ReadObject() - } - return q, err -} diff --git a/experimental/schema/query_test.go b/experimental/schema/query_test.go deleted file mode 100644 index b311fe351..000000000 --- a/experimental/schema/query_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package schema - -import ( - "encoding/json" - "fmt" - "os" - "testing" - - "github.com/invopop/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCommonSupport(t *testing.T) { - r := new(jsonschema.Reflector) - r.DoNotReference = true - err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/query", "./") - require.NoError(t, err) - - query := r.Reflect(&CommonQueryProperties{}) - query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi - query.Description = "Query properties shared by all data sources" - - // Write the map of values ignored by the common parser - fmt.Printf("var commonKeys = map[string]bool{\n") - for pair := query.Properties.Oldest(); pair != nil; pair = pair.Next() { - fmt.Printf(" \"%s\": true,\n", pair.Key) - } - fmt.Printf("}\n") - - // // Hide this old property - query.Properties.Delete("datasourceId") - out, err := json.MarshalIndent(query, "", " ") - require.NoError(t, err) - - update := false - outfile := "query.schema.json" - body, err := os.ReadFile(outfile) - if err == nil { - if !assert.JSONEq(t, string(out), string(body)) { - update = true - } - } else { - update = true - } - if update { - err = os.WriteFile(outfile, out, 0600) - require.NoError(t, err, "error writing file") - } -} diff --git a/experimental/schema/settings.go b/experimental/schema/settings.go deleted file mode 100644 index 81db055fd..000000000 --- a/experimental/schema/settings.go +++ /dev/null @@ -1,23 +0,0 @@ -package schema - -type SettingsDefinitionSpec struct { - // Multiple schemas can be defined using discriminators - Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` - - // Describe whe the query type is for - Description string `json:"description,omitempty"` - - // The query schema represents the properties that can be sent to the API - // In many cases, this may be the same properties that are saved in a dashboard - // In the case where the save model is different, we must also specify a save model - JSONDataSchema any `json:"jsonDataSchema"` - - // JSON schema defining the properties needed in secure json - // NOTE these must all be string fields - SecureJSONSchema any `json:"secureJsonSchema"` - - // Changelog defines the changed from the previous version - // All changes in the same version *must* be backwards compatible - // Only notable changes will be shown here, for the full version history see git! - Changelog []string `json:"changelog,omitempty"` -} diff --git a/experimental/testdata/folder.golden.txt b/experimental/testdata/folder.golden.txt index b6f411a27..5a2d86ffb 100644 --- a/experimental/testdata/folder.golden.txt +++ b/experimental/testdata/folder.golden.txt @@ -29,4 +29,4 @@ Dimensions: 2 Fields by 20 Rows ====== TEST DATA RESPONSE (arrow base64) ====== -FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAIAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAA/wAAAAAAAABYAQAAAAAAAAAAAAAAAAAAWAEAAAAAAABUAAAAAAAAALABAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA8QAAAPcAAAD/AAAAAAAAAFJFQURNRS5tZGFjdGlvbnNhdXRoY2xpZW50ZGF0YXNvdXJjZXRlc3RlMmVlcnJvcnNvdXJjZWZlYXR1cmV0b2dnbGVzZmlsZWluZm8uZ29maWxlaW5mb190ZXN0LmdvZnJhbWVfc29ydGVyLmdvZnJhbWVfc29ydGVyX3Rlc3QuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlci5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyX3Rlc3QuZ29odHRwX2xvZ2dlcm1hY3Jvc21vY2tvYXV0aHRva2VucmV0cmlldmVycmVzdF9jbGllbnQuZ29zY2hlbWF0ZXN0ZGF0YQAAAAAAAAAAAAkAAAASAAAAGwAAACQAAAAtAAAANgAAADYAAAA2AAAANgAAADYAAAA2AAAANgAAAD8AAABIAAAAUQAAAFoAAABaAAAAYwAAAGwAAAAAAAAAZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5AAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAAEAAEAAADYAQAAAAAAAOAAAAAAAAAAIAIAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAPgBAABBUlJPVzE= +FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAIAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAA/QAAAAAAAABYAQAAAAAAAAAAAAAAAAAAWAEAAAAAAABUAAAAAAAAALABAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA8QAAAPUAAAD9AAAAAAAAAFJFQURNRS5tZGFjdGlvbnNhdXRoY2xpZW50ZGF0YXNvdXJjZXRlc3RlMmVlcnJvcnNvdXJjZWZlYXR1cmV0b2dnbGVzZmlsZWluZm8uZ29maWxlaW5mb190ZXN0LmdvZnJhbWVfc29ydGVyLmdvZnJhbWVfc29ydGVyX3Rlc3QuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlci5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyX3Rlc3QuZ29odHRwX2xvZ2dlcm1hY3Jvc21vY2tvYXV0aHRva2VucmV0cmlldmVycmVzdF9jbGllbnQuZ29zcGVjdGVzdGRhdGEAAAAAAAAAAAAAAAkAAAASAAAAGwAAACQAAAAtAAAANgAAADYAAAA2AAAANgAAADYAAAA2AAAANgAAAD8AAABIAAAAUQAAAFoAAABaAAAAYwAAAGwAAAAAAAAAZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5AAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAAEAAEAAADYAQAAAAAAAOAAAAAAAAAAIAIAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAPgBAABBUlJPVzE= From 1202776612ce27286b6900591c56145170984beb Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 21 Feb 2024 16:35:13 -0500 Subject: [PATCH 16/71] rename to spec --- experimental/spec/builder.go | 485 ++++++++++++++++++ experimental/spec/enums.go | 112 ++++ experimental/spec/enums_test.go | 22 + experimental/spec/example/math.go | 46 ++ .../spec/example/query.post.schema.json | 199 +++++++ .../spec/example/query.save.schema.json | 172 +++++++ experimental/spec/example/query.types.json | 193 +++++++ experimental/spec/example/reduce.go | 57 ++ experimental/spec/example/resample.go | 28 + experimental/spec/example/types.go | 64 +++ experimental/spec/example/types_test.go | 64 +++ experimental/spec/k8s.go | 52 ++ experimental/spec/query.go | 157 ++++++ experimental/spec/query.schema.json | 82 +++ experimental/spec/query_parser.go | 57 ++ experimental/spec/query_test.go | 50 ++ experimental/spec/settings.go | 23 + 17 files changed, 1863 insertions(+) create mode 100644 experimental/spec/builder.go create mode 100644 experimental/spec/enums.go create mode 100644 experimental/spec/enums_test.go create mode 100644 experimental/spec/example/math.go create mode 100644 experimental/spec/example/query.post.schema.json create mode 100644 experimental/spec/example/query.save.schema.json create mode 100644 experimental/spec/example/query.types.json create mode 100644 experimental/spec/example/reduce.go create mode 100644 experimental/spec/example/resample.go create mode 100644 experimental/spec/example/types.go create mode 100644 experimental/spec/example/types_test.go create mode 100644 experimental/spec/k8s.go create mode 100644 experimental/spec/query.go create mode 100644 experimental/spec/query.schema.json create mode 100644 experimental/spec/query_parser.go create mode 100644 experimental/spec/query_test.go create mode 100644 experimental/spec/settings.go diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go new file mode 100644 index 000000000..a4a7f45de --- /dev/null +++ b/experimental/spec/builder.go @@ -0,0 +1,485 @@ +package spec + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The k8s compatible jsonschema version +const draft04 = "https://json-schema.org/draft-04/schema" + +// SchemaBuilder is a helper function that can be used by +// backend build processes to produce static schema definitions +// This is not intended as runtime code, and is not the only way to +// produce a schema (we may also want/need to use typescript as the source) +type Builder struct { + opts BuilderOptions + reflector *jsonschema.Reflector // Needed to use comments + query []QueryTypeDefinition + setting []SettingsDefinition +} + +type BuilderOptions struct { + // ex "github.com/invopop/jsonschema" + BasePackage string + + // ex "./" + CodePath string + + // explicitly define the enumeration fields + Enums []reflect.Type +} + +type QueryTypeInfo struct { + // The management name + Name string + // Optional discriminators + Discriminators []DiscriminatorFieldValue + // Raw GO type used for reflection + GoType reflect.Type + // Add sample queries + Examples []QueryExample +} + +type SettingTypeInfo struct { + // The management name + Name string + // Optional discriminators + Discriminators []DiscriminatorFieldValue + // Raw GO type used for reflection + GoType reflect.Type + // Map[string]string + SecureGoType reflect.Type +} + +func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { + r := new(jsonschema.Reflector) + r.DoNotReference = true + if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { + return nil, err + } + customMapper := map[reflect.Type]*jsonschema.Schema{ + reflect.TypeOf(data.Frame{}): { + Type: "object", + Extras: map[string]any{ + "x-grafana-type": "data.DataFrame", + }, + AdditionalProperties: jsonschema.TrueSchema, + }, + } + r.Mapper = func(t reflect.Type) *jsonschema.Schema { + return customMapper[t] + } + + if len(opts.Enums) > 0 { + fields, err := findEnumFields(opts.BasePackage, opts.CodePath) + if err != nil { + return nil, err + } + for _, etype := range opts.Enums { + for _, f := range fields { + if f.Name == etype.Name() && f.Package == etype.PkgPath() { + enumValueDescriptions := map[string]string{} + s := &jsonschema.Schema{ + Type: "string", + Extras: map[string]any{ + "x-enum-description": enumValueDescriptions, + }, + } + for _, val := range f.Values { + s.Enum = append(s.Enum, val.Value) + if val.Comment != "" { + enumValueDescriptions[val.Value] = val.Comment + } + } + customMapper[etype] = s + } + } + } + } + + return &Builder{ + opts: opts, + reflector: r, + }, nil +} + +func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { + for _, info := range inputs { + schema := b.reflector.ReflectFromType(info.GoType) + if schema == nil { + return fmt.Errorf("missing schema") + } + + b.enumify(schema) + + name := info.Name + if name == "" { + for _, dis := range info.Discriminators { + if name != "" { + name += "-" + } + name += dis.Value + } + if name == "" { + return fmt.Errorf("missing name or discriminators") + } + } + + // We need to be careful to only use draft-04 so that this is possible to use + // with kube-openapi + schema.Version = draft04 + schema.ID = "" + schema.Anchor = "" + + b.query = append(b.query, QueryTypeDefinition{ + ObjectMeta: ObjectMeta{ + Name: name, + }, + Spec: QueryTypeDefinitionSpec{ + Discriminators: info.Discriminators, + QuerySchema: schema, + Examples: info.Examples, + }, + }) + } + return nil +} + +func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { + for _, info := range inputs { + name := info.Name + if name == "" { + return fmt.Errorf("missing name") + } + + schema := b.reflector.ReflectFromType(info.GoType) + if schema == nil { + return fmt.Errorf("missing schema") + } + + b.enumify(schema) + + // used by kube-openapi + schema.Version = draft04 + schema.ID = "" + schema.Anchor = "" + + b.setting = append(b.setting, SettingsDefinition{ + ObjectMeta: ObjectMeta{ + Name: name, + }, + Spec: SettingsDefinitionSpec{ + Discriminators: info.Discriminators, + JSONDataSchema: schema, + }, + }) + } + return nil +} + +// whitespaceRegex is the regex for consecutive whitespaces. +var whitespaceRegex = regexp.MustCompile(`\s+`) + +func (b *Builder) enumify(s *jsonschema.Schema) { + if len(s.Enum) > 0 && s.Extras != nil { + extra, ok := s.Extras["x-enum-description"] + if !ok { + return + } + + lookup, ok := extra.(map[string]string) + if !ok { + return + } + + lines := []string{} + if s.Description != "" { + lines = append(lines, s.Description, "\n") + } + lines = append(lines, "Possible enum values:") + for _, v := range s.Enum { + c := lookup[v.(string)] + c = whitespaceRegex.ReplaceAllString(c, " ") + lines = append(lines, fmt.Sprintf(" - `%q` %s", v, c)) + } + + s.Description = strings.Join(lines, "\n") + return + } + + for pair := s.Properties.Oldest(); pair != nil; pair = pair.Next() { + b.enumify(pair.Value) + } +} + +// Update the schema definition file +// When placed in `static/schema/query.types.json` folder of a plugin distribution, +// it can be used to advertise various query types +// If the spec contents have changed, the test will fail (but still update the output) +func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) { + t.Helper() + + outfile := filepath.Join(outdir, "query.types.json") + now := time.Now().UTC() + rv := fmt.Sprintf("%d", now.UnixMilli()) + + defs := QueryTypeDefinitionList{} + byName := make(map[string]*QueryTypeDefinition) + body, err := os.ReadFile(outfile) + if err == nil { + err = json.Unmarshal(body, &defs) + if err == nil { + for i, def := range defs.Items { + byName[def.ObjectMeta.Name] = &defs.Items[i] + } + } + } + defs.Kind = "QueryTypeDefinitionList" + defs.APIVersion = "query.grafana.app/v0alpha1" + + // The updated schemas + for _, def := range b.query { + found, ok := byName[def.ObjectMeta.Name] + if !ok { + defs.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) + + defs.Items = append(defs.Items, def) + } else { + var o1, o2 interface{} + b1, _ := json.Marshal(def.Spec) + b2, _ := json.Marshal(found.Spec) + _ = json.Unmarshal(b1, &o1) + _ = json.Unmarshal(b2, &o2) + if !reflect.DeepEqual(o1, o2) { + found.ObjectMeta.ResourceVersion = rv + found.Spec = def.Spec + } + delete(byName, def.ObjectMeta.Name) + } + } + + if defs.ObjectMeta.ResourceVersion == "" { + defs.ObjectMeta.ResourceVersion = rv + } + + if len(byName) > 0 { + require.FailNow(t, "query type removed, manually update (for now)") + } + maybeUpdateFile(t, outfile, defs, body) + + // Read query info + r := new(jsonschema.Reflector) + r.DoNotReference = true + err = r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/schema", "./") + require.NoError(t, err) + + query := r.Reflect(&CommonQueryProperties{}) + query.Version = draft04 // used by kube-openapi + query.Description = "Query properties shared by all data sources" + + // Now update the query files + //---------------------------- + outfile = filepath.Join(outdir, "query.post.schema.json") + schema, err := toQuerySchema(query, defs, false) + require.NoError(t, err) + + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, schema, body) + + // Now update the query files + //---------------------------- + outfile = filepath.Join(outdir, "query.save.schema.json") + schema, err = toQuerySchema(query, defs, true) + require.NoError(t, err) + + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, schema, body) +} + +// Update the schema definition file +// When placed in `static/schema/query.schema.json` folder of a plugin distribution, +// it can be used to advertise various query types +// If the spec contents have changed, the test will fail (but still update the output) +func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) { + t.Helper() + + now := time.Now().UTC() + rv := fmt.Sprintf("%d", now.UnixMilli()) + + defs := QueryTypeDefinitionList{} + byName := make(map[string]*QueryTypeDefinition) + body, err := os.ReadFile(outfile) + if err == nil { + err = json.Unmarshal(body, &defs) + if err == nil { + for i, def := range defs.Items { + byName[def.ObjectMeta.Name] = &defs.Items[i] + } + } + } + defs.Kind = "SettingsDefinitionList" + defs.APIVersion = "common.grafana.app/v0alpha1" + + // The updated schemas + for _, def := range b.query { + found, ok := byName[def.ObjectMeta.Name] + if !ok { + defs.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) + + defs.Items = append(defs.Items, def) + } else { + var o1, o2 interface{} + b1, _ := json.Marshal(def.Spec) + b2, _ := json.Marshal(found.Spec) + _ = json.Unmarshal(b1, &o1) + _ = json.Unmarshal(b2, &o2) + if !reflect.DeepEqual(o1, o2) { + found.ObjectMeta.ResourceVersion = rv + found.Spec = def.Spec + } + delete(byName, def.ObjectMeta.Name) + } + } + + if defs.ObjectMeta.ResourceVersion == "" { + defs.ObjectMeta.ResourceVersion = rv + } + + if len(byName) > 0 { + require.FailNow(t, "query type removed, manually update (for now)") + } +} + +// Converts a set of queries into a single real schema (merged with the common properties) +func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, saveModel bool) (*jsonschema.Schema, error) { + descr := "Query model (the payload sent to /ds/query)" + if saveModel { + descr = "Save model (the payload saved in dashboards and alerts)" + } + + ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true, "timeRange": true} + definitions := make(jsonschema.Definitions) + common := make(map[string]*jsonschema.Schema) + for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { + if saveModel && ignoreForSave[pair.Key] { + continue // + } + definitions[pair.Key] = pair.Value + common[pair.Key] = &jsonschema.Schema{Ref: "#/definitions/" + pair.Key} + } + + // The types for each query type + queryTypes := []*jsonschema.Schema{} + for _, qt := range defs.Items { + node, err := asJSONSchema(qt.Spec.QuerySchema) + node.Version = "" + if err != nil { + return nil, fmt.Errorf("error reading query types schema: %s // %w", qt.ObjectMeta.Name, err) + } + if node == nil { + return nil, fmt.Errorf("missing query schema: %s // %v", qt.ObjectMeta.Name, qt) + } + + // Match all discriminators + for _, d := range qt.Spec.Discriminators { + ds, ok := node.Properties.Get(d.Field) + if !ok { + ds = &jsonschema.Schema{Type: "string"} + node.Properties.Set(d.Field, ds) + } + ds.Pattern = `^` + d.Value + `$` + node.Required = append(node.Required, d.Field) + } + + queryTypes = append(queryTypes, node) + } + + // Single node -- just union the global and local properties + if len(queryTypes) == 1 { + node := queryTypes[0] + node.Version = draft04 + node.Description = descr + for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { + _, found := node.Properties.Get(pair.Key) + if found { + continue + } + node.Properties.Set(pair.Key, pair.Value) + } + return node, nil + } + + s := &jsonschema.Schema{ + Type: "object", + Version: draft04, + Properties: jsonschema.NewProperties(), + Definitions: make(jsonschema.Definitions), + Description: descr, + } + + for _, qt := range queryTypes { + qt.Required = append(qt.Required, "refId") + + for k, v := range common { + _, found := qt.Properties.Get(k) + if found { + continue + } + qt.Properties.Set(k, v) + } + + s.OneOf = append(s.OneOf, qt) + } + return s, nil +} + +func asJSONSchema(v any) (*jsonschema.Schema, error) { + s, ok := v.(*jsonschema.Schema) + if ok { + return s, nil + } + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + s = &jsonschema.Schema{} + err = json.Unmarshal(b, s) + return s, err +} + +func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { + t.Helper() + + out, err := json.MarshalIndent(value, "", " ") + require.NoError(t, err) + + update := false + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true + } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0600) + require.NoError(t, err, "error writing file") + } +} diff --git a/experimental/spec/enums.go b/experimental/spec/enums.go new file mode 100644 index 000000000..fb1ae0f2b --- /dev/null +++ b/experimental/spec/enums.go @@ -0,0 +1,112 @@ +package spec + +import ( + "io/fs" + gopath "path" + "path/filepath" + "strings" + + "go/ast" + "go/doc" + "go/parser" + "go/token" +) + +type EnumValue struct { + Value string + Comment string +} + +type EnumField struct { + Package string + Name string + Comment string + Values []EnumValue +} + +func findEnumFields(base, path string) ([]EnumField, error) { + fset := token.NewFileSet() + dict := make(map[string][]*ast.Package) + err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + d, err := parser.ParseDir(fset, path, nil, parser.ParseComments) + if err != nil { + return err + } + for _, v := range d { + // paths may have multiple packages, like for tests + k := gopath.Join(base, path) + dict[k] = append(dict[k], v) + } + } + return nil + }) + if err != nil { + return nil, err + } + + fields := make([]EnumField, 0) + field := &EnumField{} + dp := &doc.Package{} + + for pkg, p := range dict { + for _, f := range p { + gtxt := "" + typ := "" + ast.Inspect(f, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.TypeSpec: + typ = x.Name.String() + if !ast.IsExported(typ) { + typ = "" + } else { + txt := x.Doc.Text() + if txt == "" && gtxt != "" { + txt = gtxt + gtxt = "" + } + txt = strings.TrimSpace(dp.Synopsis(txt)) + if strings.HasSuffix(txt, "+enum") { + fields = append(fields, EnumField{ + Package: pkg, + Name: typ, + Comment: strings.TrimSpace(strings.TrimSuffix(txt, "+enum")), + }) + field = &fields[len(fields)-1] + } + } + case *ast.ValueSpec: + txt := x.Doc.Text() + if txt == "" { + txt = x.Comment.Text() + } + if typ == field.Name { + for _, n := range x.Names { + if ast.IsExported(n.String()) { + v, ok := x.Values[0].(*ast.BasicLit) + if ok { + val := strings.TrimPrefix(v.Value, `"`) + val = strings.TrimSuffix(val, `"`) + txt = strings.TrimSpace(txt) + field.Values = append(field.Values, EnumValue{ + Value: val, + Comment: txt, + }) + } + } + } + } + case *ast.GenDecl: + // remember for the next type + gtxt = x.Doc.Text() + } + return true + }) + } + } + + return fields, nil +} diff --git a/experimental/spec/enums_test.go b/experimental/spec/enums_test.go new file mode 100644 index 000000000..dd9d27b5a --- /dev/null +++ b/experimental/spec/enums_test.go @@ -0,0 +1,22 @@ +package spec + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFindEnums(t *testing.T) { + fields, err := findEnumFields( + "github.com/grafana/grafana-plugin-sdk-go/experimental/spec", + "./example") + require.NoError(t, err) + + out, err := json.MarshalIndent(fields, "", " ") + require.NoError(t, err) + fmt.Printf("%s", string(out)) + + require.Equal(t, 3, len(fields)) +} diff --git a/experimental/spec/example/math.go b/experimental/spec/example/math.go new file mode 100644 index 000000000..0ef5d43c6 --- /dev/null +++ b/experimental/spec/example/math.go @@ -0,0 +1,46 @@ +package example + +import "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + +var _ ExpressionQuery = (*MathQuery)(nil) + +type MathQuery struct { + // General math expression + Expression string `json:"expression" jsonschema:"minLength=1,example=$A + 1,example=$A/$B"` + + // Parsed from the expression + variables []string `json:"-"` +} + +func (*MathQuery) ExpressionQueryType() QueryType { + return QueryTypeMath +} + +func (q *MathQuery) Variables() []string { + return q.variables +} + +func readMathQuery(iter *jsoniter.Iterator) (*MathQuery, error) { + var q *MathQuery + var err error + fname := "" + for fname, err = iter.ReadObject(); fname != "" && err == nil; fname, err = iter.ReadObject() { + switch fname { + case "expression": + temp, err := iter.ReadString() + if err != nil { + return q, err + } + q = &MathQuery{ + Expression: temp, + } + + default: + _, err = iter.ReadAny() // eat up the unused fields + if err != nil { + return nil, err + } + } + } + return q, nil +} diff --git a/experimental/spec/example/query.post.schema.json b/experimental/spec/example/query.post.schema.json new file mode 100644 index 000000000..ae7fcb575 --- /dev/null +++ b/experimental/spec/example/query.post.schema.json @@ -0,0 +1,199 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema", + "oneOf": [ + { + "properties": { + "expression": { + "type": "string", + "minLength": 1, + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ] + }, + { + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " + }, + "settings": { + "properties": { + "mode": { + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" + }, + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "mode" + ], + "description": "Reducer Options" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ] + }, + { + "properties": { + "downsampler": { + "type": "string", + "description": "The reducer" + }, + "expression": { + "type": "string", + "description": "The math expression" + }, + "loadedDimensions": { + "additionalProperties": true, + "type": "object" + }, + "upsampler": { + "type": "string", + "description": "The reducer" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "description": "QueryType = resample" + } + ], + "properties": {}, + "type": "object", + "description": "Query model (the payload sent to /ds/query)" +} \ No newline at end of file diff --git a/experimental/spec/example/query.save.schema.json b/experimental/spec/example/query.save.schema.json new file mode 100644 index 000000000..256deaa88 --- /dev/null +++ b/experimental/spec/example/query.save.schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema", + "oneOf": [ + { + "properties": { + "expression": { + "type": "string", + "minLength": 1, + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "datasource": { + "$ref": "#/definitions/datasource" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ] + }, + { + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " + }, + "settings": { + "properties": { + "mode": { + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" + }, + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "mode" + ], + "description": "Reducer Options" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "hide": { + "$ref": "#/definitions/hide" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ] + }, + { + "properties": { + "downsampler": { + "type": "string", + "description": "The reducer" + }, + "expression": { + "type": "string", + "description": "The math expression" + }, + "loadedDimensions": { + "additionalProperties": true, + "type": "object" + }, + "upsampler": { + "type": "string", + "description": "The reducer" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "hide": { + "$ref": "#/definitions/hide" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "description": "QueryType = resample" + } + ], + "properties": {}, + "type": "object", + "description": "Save model (the payload saved in dashboards and alerts)" +} \ No newline at end of file diff --git a/experimental/spec/example/query.types.json b/experimental/spec/example/query.types.json new file mode 100644 index 000000000..cc1cfceb5 --- /dev/null +++ b/experimental/spec/example/query.types.json @@ -0,0 +1,193 @@ +{ + "kind": "QueryTypeDefinitionList", + "apiVersion": "query.grafana.app/v0alpha1", + "metadata": { + "resourceVersion": "1708548629808" + }, + "items": [ + { + "metadata": { + "name": "math", + "resourceVersion": "1708548629808", + "creationTimestamp": "2024-02-21T20:50:29Z" + }, + "spec": { + "discriminators": [ + { + "field": "queryType", + "value": "math" + } + ], + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", + "properties": { + "expression": { + "type": "string", + "minLength": 1, + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression" + ] + }, + "examples": [ + { + "name": "constant addition", + "queryPayload": { + "expression": "$A + 10" + } + }, + { + "name": "math with two queries", + "queryPayload": { + "expression": "$A - $B" + } + } + ] + } + }, + { + "metadata": { + "name": "reduce", + "resourceVersion": "1708548629808", + "creationTimestamp": "2024-02-21T20:50:29Z" + }, + "spec": { + "discriminators": [ + { + "field": "queryType", + "value": "reduce" + } + ], + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" + } + }, + "settings": { + "properties": { + "mode": { + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "x-enum-description": { + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" + } + }, + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "mode" + ], + "description": "Reducer Options" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings" + ] + }, + "examples": [ + { + "name": "get max value", + "queryPayload": { + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + } + ] + } + }, + { + "metadata": { + "name": "resample", + "resourceVersion": "1708548629808", + "creationTimestamp": "2024-02-21T20:50:29Z" + }, + "spec": { + "discriminators": [ + { + "field": "queryType", + "value": "resample" + } + ], + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", + "properties": { + "expression": { + "type": "string", + "description": "The math expression" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, + "downsampler": { + "type": "string", + "description": "The reducer" + }, + "upsampler": { + "type": "string", + "description": "The reducer" + }, + "loadedDimensions": { + "additionalProperties": true, + "type": "object", + "x-grafana-type": "data.DataFrame" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions" + ], + "description": "QueryType = resample" + } + } + } + ] +} \ No newline at end of file diff --git a/experimental/spec/example/reduce.go b/experimental/spec/example/reduce.go new file mode 100644 index 000000000..3893cc06c --- /dev/null +++ b/experimental/spec/example/reduce.go @@ -0,0 +1,57 @@ +package example + +var _ ExpressionQuery = (*ReduceQuery)(nil) + +type ReduceQuery struct { + // Reference to other query results + Expression string `json:"expression"` + + // The reducer + Reducer ReducerID `json:"reducer"` + + // Reducer Options + Settings ReduceSettings `json:"settings"` +} + +func (*ReduceQuery) ExpressionQueryType() QueryType { + return QueryTypeReduce +} + +func (q *ReduceQuery) Variables() []string { + return []string{q.Expression} +} + +type ReduceSettings struct { + // Non-number reduce behavior + Mode ReduceMode `json:"mode"` + + // Only valid when mode is replace + ReplaceWithValue *float64 `json:"replaceWithValue,omitempty"` +} + +// The reducer function +// +enum +type ReducerID string + +const ( + // The sum + ReducerSum ReducerID = "sum" + // The mean + ReducerMean ReducerID = "mean" + ReducerMin ReducerID = "min" + ReducerMax ReducerID = "max" + ReducerCount ReducerID = "count" + ReducerLast ReducerID = "last" +) + +// Non-Number behavior mode +// +enum +type ReduceMode string + +const ( + // Drop non-numbers + ReduceModeDrop ReduceMode = "dropNN" + + // Replace non-numbers + ReduceModeReplace ReduceMode = "replaceNN" +) diff --git a/experimental/spec/example/resample.go b/experimental/spec/example/resample.go new file mode 100644 index 000000000..77f27b1c6 --- /dev/null +++ b/experimental/spec/example/resample.go @@ -0,0 +1,28 @@ +package example + +import "github.com/grafana/grafana-plugin-sdk-go/data" + +// QueryType = resample +type ResampleQuery struct { + // The math expression + Expression string `json:"expression"` + + // A time duration string + Window string `json:"window"` + + // The reducer + Downsampler string `json:"downsampler"` + + // The reducer + Upsampler string `json:"upsampler"` + + LoadedDimensions *data.Frame `json:"loadedDimensions"` +} + +func (*ResampleQuery) ExpressionQueryType() QueryType { + return QueryTypeReduce +} + +func (q *ResampleQuery) Variables() []string { + return []string{q.Expression} +} diff --git a/experimental/spec/example/types.go b/experimental/spec/example/types.go new file mode 100644 index 000000000..fba0c0c1d --- /dev/null +++ b/experimental/spec/example/types.go @@ -0,0 +1,64 @@ +package example + +import ( + "embed" + "encoding/json" + "fmt" + + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + "github.com/grafana/grafana-plugin-sdk-go/experimental/spec" +) + +// Supported expression types +// +enum +type QueryType string + +const ( + // Math query type + QueryTypeMath QueryType = "math" + + // Reduce query type + QueryTypeReduce QueryType = "reduce" + + // Reduce query type + QueryTypeResample QueryType = "resample" +) + +type ExpressionQuery interface { + ExpressionQueryType() QueryType + Variables() []string +} + +var _ spec.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) + +type QueyHandler struct{} + +//go:embed query.types.json +var f embed.FS + +func (*QueyHandler) QueryTypeDefinitionsJSON() (json.RawMessage, error) { + return f.ReadFile("query.types.json") +} + +// ReadQuery implements query.TypedQueryHandler. +func (*QueyHandler) ParseQuery( + // Properties that have been parsed off the same node + common spec.CommonQueryProperties, + // An iterator with context for the full node (include common values) + iter *jsoniter.Iterator, +) (ExpressionQuery, error) { + qt := QueryType(common.QueryType) + switch qt { + case QueryTypeMath: + return readMathQuery(iter) + + case QueryTypeReduce: + q := &ReduceQuery{} + err := iter.ReadVal(q) + return q, err + + case QueryTypeResample: + return nil, nil + } + return nil, fmt.Errorf("unknown query type") +} diff --git a/experimental/spec/example/types_test.go b/experimental/spec/example/types_test.go new file mode 100644 index 000000000..6982eb6a4 --- /dev/null +++ b/experimental/spec/example/types_test.go @@ -0,0 +1,64 @@ +package example + +import ( + "reflect" + "testing" + + schema "github.com/grafana/grafana-plugin-sdk-go/experimental/spec" + "github.com/stretchr/testify/require" +) + +func TestQueryTypeDefinitions(t *testing.T) { + builder, err := schema.NewSchemaBuilder(schema.BuilderOptions{ + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/spec/example", + CodePath: "./", + // We need to identify the enum fields explicitly :( + // *AND* have the +enum common for this to work + Enums: []reflect.Type{ + reflect.TypeOf(ReducerSum), // pick an example value (not the root) + reflect.TypeOf(ReduceModeDrop), // pick an example value (not the root) + }, + }) + require.NoError(t, err) + err = builder.AddQueries(schema.QueryTypeInfo{ + Discriminators: schema.NewDiscriminators("queryType", QueryTypeMath), + GoType: reflect.TypeOf(&MathQuery{}), + Examples: []schema.QueryExample{ + { + Name: "constant addition", + QueryPayload: MathQuery{ + Expression: "$A + 10", + }, + }, + { + Name: "math with two queries", + QueryPayload: MathQuery{ + Expression: "$A - $B", + }, + }, + }, + }, + schema.QueryTypeInfo{ + Discriminators: schema.NewDiscriminators("queryType", QueryTypeReduce), + GoType: reflect.TypeOf(&ReduceQuery{}), + Examples: []schema.QueryExample{ + { + Name: "get max value", + QueryPayload: ReduceQuery{ + Expression: "$A", + Reducer: ReducerMax, + Settings: ReduceSettings{ + Mode: ReduceModeDrop, + }, + }, + }, + }, + }, + schema.QueryTypeInfo{ + Discriminators: schema.NewDiscriminators("queryType", QueryTypeResample), + GoType: reflect.TypeOf(&ResampleQuery{}), + }) + require.NoError(t, err) + + builder.UpdateQueryDefinition(t, "./") +} diff --git a/experimental/spec/k8s.go b/experimental/spec/k8s.go new file mode 100644 index 000000000..30c5bdf7e --- /dev/null +++ b/experimental/spec/k8s.go @@ -0,0 +1,52 @@ +package spec + +// ObjectMeta is a struct that aims to "look" like a real kubernetes object when +// written to JSON, however it does not require the pile of dependencies +// This is really an internal helper until we decide which dependencies make sense +// to require within the SDK +type ObjectMeta struct { + // The name is for k8s and description, but not used in the schema + Name string `json:"name,omitempty"` + // Changes indicate that *something * changed + ResourceVersion string `json:"resourceVersion,omitempty"` + // Timestamp + CreationTimestamp string `json:"creationTimestamp,omitempty"` +} + +// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition +type QueryTypeDefinition struct { + ObjectMeta ObjectMeta `json:"metadata,omitempty"` + + Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` +} + +// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types +// For simple data sources, there may be only a single query type, however when multiple types +// exist they must be clearly specified with distinct discriminator field+value pairs +type QueryTypeDefinitionList struct { + Kind string `json:"kind"` // "QueryTypeDefinitionList", + APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", + + ObjectMeta `json:"metadata,omitempty"` + + Items []QueryTypeDefinition `json:"items"` +} + +// SettingsDefinition is a kubernetes shaped object that represents a single query definition +type SettingsDefinition struct { + ObjectMeta ObjectMeta `json:"metadata,omitempty"` + + Spec SettingsDefinitionSpec `json:"spec,omitempty"` +} + +// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types +// For simple data sources, there may be only a single query type, however when multiple types +// exist they must be clearly specified with distinct discriminator field+value pairs +type SettingsDefinitionList struct { + Kind string `json:"kind"` // "SettingsDefinitionList", + APIVersion string `json:"apiVersion"` // "??.common.grafana.app/v0alpha1", + + ObjectMeta `json:"metadata,omitempty"` + + Items []SettingsDefinition `json:"items"` +} diff --git a/experimental/spec/query.go b/experimental/spec/query.go new file mode 100644 index 000000000..2a1acc21f --- /dev/null +++ b/experimental/spec/query.go @@ -0,0 +1,157 @@ +package spec + +import ( + "embed" + "encoding/json" + "fmt" + + "github.com/grafana/grafana-plugin-sdk-go/data" +) + +type DiscriminatorFieldValue struct { + // DiscriminatorField is the field used to link behavior to this specific + // query type. It is typically "queryType", but can be another field if necessary + Field string `json:"field"` + + // The discriminator value + Value string `json:"value"` +} + +// using any since this will often be enumerations +func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { + if len(keyvals)%2 != 0 { + panic("values must be even") + } + dis := []DiscriminatorFieldValue{} + for i := 0; i < len(keyvals); i += 2 { + dis = append(dis, DiscriminatorFieldValue{ + Field: fmt.Sprintf("%v", keyvals[i]), + Value: fmt.Sprintf("%v", keyvals[i+1]), + }) + } + return dis +} + +type QueryTypeDefinitionSpec struct { + // Multiple schemas can be defined using discriminators + Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // The query schema represents the properties that can be sent to the API + // In many cases, this may be the same properties that are saved in a dashboard + // In the case where the save model is different, we must also specify a save model + QuerySchema any `json:"querySchema"` + + // The save model defines properties that can be saved into dashboard or similar + // These values are processed by frontend components and then sent to the api + // When specified, this schema will be used to validate saved objects rather than + // the query schema + SaveModel any `json:"saveModel,omitempty"` + + // Examples (include a wrapper) ideally a template! + Examples []QueryExample `json:"examples,omitempty"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} + +type QueryExample struct { + // Version identifier or empty if only one exists + Name string `json:"name,omitempty"` + + // An example payload -- this should not require the frontend code to + // pre-process anything + QueryPayload any `json:"queryPayload,omitempty"` + + // An example save model -- this will require frontend code to convert it + // into a valid query payload + SaveModel any `json:"saveModel,omitempty"` +} + +type CommonQueryProperties struct { + // RefID is the unique identifier of the query, set by the frontend call. + RefID string `json:"refId,omitempty"` + + // Optionally define expected query result behavior + ResultAssertions *ResultAssertions `json:"resultAssertions,omitempty"` + + // TimeRange represents the query range + // NOTE: unlike generic /ds/query, we can now send explicit time values in each query + // NOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly + TimeRange *TimeRange `json:"timeRange,omitempty"` + + // The datasource + Datasource *DataSourceRef `json:"datasource,omitempty"` + + // Deprecated -- use datasource ref instead + DatasourceID int64 `json:"datasourceId,omitempty"` + + // QueryType is an optional identifier for the type of query. + // It can be used to distinguish different types of queries. + QueryType string `json:"queryType,omitempty"` + + // MaxDataPoints is the maximum number of data points that should be returned from a time series query. + // NOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated + // from the number of pixels visible in a visualization + MaxDataPoints int64 `json:"maxDataPoints,omitempty"` + + // Interval is the suggested duration between time points in a time series query. + // NOTE: the values for intervalMs is not saved in the query model. It is typically calculated + // from the interval required to fill a pixels in the visualization + IntervalMS float64 `json:"intervalMs,omitempty"` + + // true if query is disabled (ie should not be returned to the dashboard) + // NOTE: this does not always imply that the query should not be executed since + // the results from a hidden query may be used as the input to other queries (SSE etc) + Hide bool `json:"hide,omitempty"` +} + +type DataSourceRef struct { + // The datasource plugin type + Type string `json:"type"` + + // Datasource UID + UID string `json:"uid"` + + // ?? the datasource API version? (just version, not the group? type | apiVersion?) +} + +// TimeRange represents a time range for a query and is a property of DataQuery. +type TimeRange struct { + // From is the start time of the query. + From string `json:"from"` + + // To is the end time of the query. + To string `json:"to"` +} + +// ResultAssertions define the expected response shape and query behavior. This is useful to +// enforce behavior over time. The assertions are passed to the query engine and can be used +// to fail queries *before* returning them to a client (select * from bigquery!) +type ResultAssertions struct { + // Type asserts that the frame matches a known type structure. + Type data.FrameType `json:"type,omitempty"` + + // TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane + // contract documentation https://grafana.github.io/dataplane/contract/. + TypeVersion data.FrameTypeVersion `json:"typeVersion"` + + // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast + MaxBytes int64 `json:"maxBytes,omitempty"` + + // Maximum frame count + MaxFrames int64 `json:"maxFrames,omitempty"` +} + +//go:embed query.schema.json +var f embed.FS + +// Get the cached feature list (exposed as a k8s resource) +func GetCommonJSONSchema() json.RawMessage { + body, _ := f.ReadFile("common.jsonschema") + return body +} diff --git a/experimental/spec/query.schema.json b/experimental/spec/query.schema.json new file mode 100644 index 000000000..3c66d539a --- /dev/null +++ b/experimental/spec/query.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema", + "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/spec/common-query-properties", + "properties": { + "refId": { + "type": "string" + }, + "resultAssertions": { + "properties": { + "type": { + "type": "string" + }, + "typeVersion": { + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2 + }, + "maxBytes": { + "type": "integer" + }, + "maxFrames": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "typeVersion" + ] + }, + "timeRange": { + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "from", + "to" + ] + }, + "datasource": { + "properties": { + "type": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type", + "uid" + ] + }, + "queryType": { + "type": "string" + }, + "maxDataPoints": { + "type": "integer" + }, + "intervalMs": { + "type": "number" + }, + "hide": { + "type": "boolean" + } + }, + "additionalProperties": false, + "type": "object", + "description": "Query properties shared by all data sources" +} \ No newline at end of file diff --git a/experimental/spec/query_parser.go b/experimental/spec/query_parser.go new file mode 100644 index 000000000..54278501b --- /dev/null +++ b/experimental/spec/query_parser.go @@ -0,0 +1,57 @@ +package spec + +import ( + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" +) + +// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing +type GenericDataQuery struct { + CommonQueryProperties `json:",inline"` + + // Additional Properties (that live at the root) + Additional map[string]any `json:",inline"` +} + +// Generic query parser pattern. +type TypedQueryParser[Q any] interface { + // Get the query parser for a query type + // The version is split from the end of the discriminator field + ParseQuery( + // Properties that have been parsed off the same node + common CommonQueryProperties, + // An iterator with context for the full node (include common values) + iter *jsoniter.Iterator, + ) (Q, error) +} + +var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) + +type GenericQueryParser struct{} + +var commonKeys = map[string]bool{ + "refId": true, + "resultAssertions": true, + "timeRange": true, + "datasource": true, + "datasourceId": true, + "queryType": true, + "maxDataPoints": true, + "intervalMs": true, + "hide": true, +} + +// ParseQuery implements TypedQueryParser. +func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator) (GenericDataQuery, error) { + q := GenericDataQuery{CommonQueryProperties: common, Additional: make(map[string]any)} + field, err := iter.ReadObject() + for field != "" && err == nil { + if !commonKeys[field] { + q.Additional[field], err = iter.Read() + if err != nil { + return q, err + } + } + field, err = iter.ReadObject() + } + return q, err +} diff --git a/experimental/spec/query_test.go b/experimental/spec/query_test.go new file mode 100644 index 000000000..e90726d43 --- /dev/null +++ b/experimental/spec/query_test.go @@ -0,0 +1,50 @@ +package spec + +import ( + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCommonSupport(t *testing.T) { + r := new(jsonschema.Reflector) + r.DoNotReference = true + err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/query", "./") + require.NoError(t, err) + + query := r.Reflect(&CommonQueryProperties{}) + query.Version = draft04 // used by kube-openapi + query.Description = "Query properties shared by all data sources" + + // Write the map of values ignored by the common parser + fmt.Printf("var commonKeys = map[string]bool{\n") + for pair := query.Properties.Oldest(); pair != nil; pair = pair.Next() { + fmt.Printf(" \"%s\": true,\n", pair.Key) + } + fmt.Printf("}\n") + + // // Hide this old property + query.Properties.Delete("datasourceId") + out, err := json.MarshalIndent(query, "", " ") + require.NoError(t, err) + + update := false + outfile := "query.schema.json" + body, err := os.ReadFile(outfile) + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true + } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0600) + require.NoError(t, err, "error writing file") + } +} diff --git a/experimental/spec/settings.go b/experimental/spec/settings.go new file mode 100644 index 000000000..30e092d37 --- /dev/null +++ b/experimental/spec/settings.go @@ -0,0 +1,23 @@ +package spec + +type SettingsDefinitionSpec struct { + // Multiple schemas can be defined using discriminators + Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // The query schema represents the properties that can be sent to the API + // In many cases, this may be the same properties that are saved in a dashboard + // In the case where the save model is different, we must also specify a save model + JSONDataSchema any `json:"jsonDataSchema"` + + // JSON schema defining the properties needed in secure json + // NOTE these must all be string fields + SecureJSONSchema any `json:"secureJsonSchema"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} From 10b145dd3c45ac25f8048358164099ef6d747687 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sat, 24 Feb 2024 15:49:53 -0800 Subject: [PATCH 17/71] more reference stuff --- experimental/spec/builder.go | 61 +++++- .../spec/example/{types.go => query.go} | 0 .../spec/example/query.post.schema.json | 132 ++++++++--- .../spec/example/query.save.schema.json | 74 ++++++- experimental/spec/example/query.types.json | 46 ++-- .../example/{types_test.go => query_test.go} | 38 ++-- .../testdata/sample_query_request.json | 22 ++ experimental/spec/query.go | 9 +- experimental/spec/query.schema.json | 48 ++-- experimental/spec/query_parser.go | 205 +++++++++++++++++- experimental/spec/query_test.go | 2 +- 11 files changed, 523 insertions(+), 114 deletions(-) rename experimental/spec/example/{types.go => query.go} (100%) rename experimental/spec/example/{types_test.go => query_test.go} (56%) create mode 100644 experimental/spec/example/testdata/sample_query_request.json diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go index a4a7f45de..b06505bc4 100644 --- a/experimental/spec/builder.go +++ b/experimental/spec/builder.go @@ -95,7 +95,7 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { s := &jsonschema.Schema{ Type: "string", Extras: map[string]any{ - "x-enum-description": enumValueDescriptions, + "x-enum-dictionary": enumValueDescriptions, }, } for _, val := range f.Values { @@ -195,7 +195,7 @@ var whitespaceRegex = regexp.MustCompile(`\s+`) func (b *Builder) enumify(s *jsonschema.Schema) { if len(s.Enum) > 0 && s.Extras != nil { - extra, ok := s.Extras["x-enum-description"] + extra, ok := s.Extras["x-enum-dictionary"] if !ok { return } @@ -229,7 +229,7 @@ func (b *Builder) enumify(s *jsonschema.Schema) { // When placed in `static/schema/query.types.json` folder of a plugin distribution, // it can be used to advertise various query types // If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) { +func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) QueryTypeDefinitionList { t.Helper() outfile := filepath.Join(outdir, "query.types.json") @@ -282,10 +282,14 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) { } maybeUpdateFile(t, outfile, defs, body) + // Make sure the sample queries are actually valid + _, err = GetExampleQueries(defs) + require.NoError(t, err) + // Read query info r := new(jsonschema.Reflector) r.DoNotReference = true - err = r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/schema", "./") + err = r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/spec", "./") require.NoError(t, err) query := r.Reflect(&CommonQueryProperties{}) @@ -309,20 +313,21 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) { body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, schema, body) + return defs } // Update the schema definition file // When placed in `static/schema/query.schema.json` folder of a plugin distribution, // it can be used to advertise various query types // If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) { +func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) SettingsDefinitionList { t.Helper() now := time.Now().UTC() rv := fmt.Sprintf("%d", now.UnixMilli()) - defs := QueryTypeDefinitionList{} - byName := make(map[string]*QueryTypeDefinition) + defs := SettingsDefinitionList{} + byName := make(map[string]*SettingsDefinition) body, err := os.ReadFile(outfile) if err == nil { err = json.Unmarshal(body, &defs) @@ -336,7 +341,7 @@ func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) { defs.APIVersion = "common.grafana.app/v0alpha1" // The updated schemas - for _, def := range b.query { + for _, def := range b.setting { found, ok := byName[def.ObjectMeta.Name] if !ok { defs.ObjectMeta.ResourceVersion = rv @@ -363,8 +368,9 @@ func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) { } if len(byName) > 0 { - require.FailNow(t, "query type removed, manually update (for now)") + require.FailNow(t, "settings type removed, manually update (for now)") } + return defs } // Converts a set of queries into a single real schema (merged with the common properties) @@ -416,6 +422,7 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav node := queryTypes[0] node.Version = draft04 node.Description = descr + node.Definitions = definitions for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { _, found := node.Properties.Get(pair.Key) if found { @@ -430,7 +437,7 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav Type: "object", Version: draft04, Properties: jsonschema.NewProperties(), - Definitions: make(jsonschema.Definitions), + Definitions: definitions, Description: descr, } @@ -464,6 +471,20 @@ func asJSONSchema(v any) (*jsonschema.Schema, error) { return s, err } +func asGenericDataQuery(v any) (*GenericDataQuery, error) { + s, ok := v.(*GenericDataQuery) + if ok { + return s, nil + } + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + s = &GenericDataQuery{} + err = json.Unmarshal(b, s) + return s, err +} + func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { t.Helper() @@ -483,3 +504,23 @@ func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { require.NoError(t, err, "error writing file") } } + +func GetExampleQueries(defs QueryTypeDefinitionList) (QueryRequest[GenericDataQuery], error) { + rsp := QueryRequest[GenericDataQuery]{ + Queries: []GenericDataQuery{}, + } + for _, def := range defs.Items { + for _, sample := range def.Spec.Examples { + if sample.SaveModel != nil { + q, err := asGenericDataQuery(sample.SaveModel) + if err != nil { + return rsp, fmt.Errorf("invalid sample save query [%s], in %s // %w", + sample.Name, def.ObjectMeta.Name, err) + } + q.RefID = string(rune('A' + len(rsp.Queries))) + rsp.Queries = append(rsp.Queries, *q) + } + } + } + return rsp, nil +} diff --git a/experimental/spec/example/types.go b/experimental/spec/example/query.go similarity index 100% rename from experimental/spec/example/types.go rename to experimental/spec/example/query.go diff --git a/experimental/spec/example/query.post.schema.json b/experimental/spec/example/query.post.schema.json index ae7fcb575..8ff42e65f 100644 --- a/experimental/spec/example/query.post.schema.json +++ b/experimental/spec/example/query.post.schema.json @@ -1,5 +1,83 @@ { "$schema": "https://json-schema.org/draft-04/schema", + "$defs": { + "datasource": { + "properties": { + "type": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type", + "uid" + ] + }, + "datasourceId": { + "type": "integer" + }, + "hide": { + "type": "boolean" + }, + "intervalMs": { + "type": "number" + }, + "maxDataPoints": { + "type": "integer" + }, + "queryType": { + "type": "string" + }, + "refId": { + "type": "string" + }, + "resultAssertions": { + "properties": { + "type": { + "type": "string" + }, + "typeVersion": { + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2 + }, + "maxBytes": { + "type": "integer" + }, + "maxFrames": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "typeVersion" + ] + }, + "timeRange": { + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "from", + "to" + ] + } + }, "oneOf": [ { "properties": { @@ -16,20 +94,14 @@ "type": "string", "pattern": "^math$" }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - }, "intervalMs": { "$ref": "#/definitions/intervalMs" }, "hide": { "$ref": "#/definitions/hide" }, - "refId": { - "$ref": "#/definitions/refId" + "datasource": { + "$ref": "#/definitions/datasource" }, "resultAssertions": { "$ref": "#/definitions/resultAssertions" @@ -37,8 +109,14 @@ "timeRange": { "$ref": "#/definitions/timeRange" }, - "datasource": { - "$ref": "#/definitions/datasource" + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + }, + "refId": { + "$ref": "#/definitions/refId" } }, "additionalProperties": false, @@ -93,29 +171,29 @@ "type": "string", "pattern": "^reduce$" }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "intervalMs": { + "$ref": "#/definitions/intervalMs" }, - "timeRange": { - "$ref": "#/definitions/timeRange" + "hide": { + "$ref": "#/definitions/hide" }, "datasource": { "$ref": "#/definitions/datasource" }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" }, - "hide": { - "$ref": "#/definitions/hide" + "timeRange": { + "$ref": "#/definitions/timeRange" }, "datasourceId": { "$ref": "#/definitions/datasourceId" }, "maxDataPoints": { "$ref": "#/definitions/maxDataPoints" + }, + "refId": { + "$ref": "#/definitions/refId" } }, "additionalProperties": false, @@ -163,6 +241,12 @@ "timeRange": { "$ref": "#/definitions/timeRange" }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + }, "datasource": { "$ref": "#/definitions/datasource" }, @@ -171,12 +255,6 @@ }, "hide": { "$ref": "#/definitions/hide" - }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" } }, "additionalProperties": false, diff --git a/experimental/spec/example/query.save.schema.json b/experimental/spec/example/query.save.schema.json index 256deaa88..816f64f1d 100644 --- a/experimental/spec/example/query.save.schema.json +++ b/experimental/spec/example/query.save.schema.json @@ -1,5 +1,61 @@ { "$schema": "https://json-schema.org/draft-04/schema", + "$defs": { + "datasource": { + "properties": { + "type": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type", + "uid" + ] + }, + "datasourceId": { + "type": "integer" + }, + "hide": { + "type": "boolean" + }, + "queryType": { + "type": "string" + }, + "refId": { + "type": "string" + }, + "resultAssertions": { + "properties": { + "type": { + "type": "string" + }, + "typeVersion": { + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2 + }, + "maxBytes": { + "type": "integer" + }, + "maxFrames": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "typeVersion" + ] + } + }, "oneOf": [ { "properties": { @@ -16,9 +72,6 @@ "type": "string", "pattern": "^math$" }, - "datasourceId": { - "$ref": "#/definitions/datasourceId" - }, "hide": { "$ref": "#/definitions/hide" }, @@ -30,6 +83,9 @@ }, "datasource": { "$ref": "#/definitions/datasource" + }, + "datasourceId": { + "$ref": "#/definitions/datasourceId" } }, "additionalProperties": false, @@ -136,12 +192,6 @@ "type": "string", "pattern": "^resample$" }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, "datasource": { "$ref": "#/definitions/datasource" }, @@ -150,6 +200,12 @@ }, "hide": { "$ref": "#/definitions/hide" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" } }, "additionalProperties": false, diff --git a/experimental/spec/example/query.types.json b/experimental/spec/example/query.types.json index cc1cfceb5..5a5982001 100644 --- a/experimental/spec/example/query.types.json +++ b/experimental/spec/example/query.types.json @@ -8,7 +8,7 @@ { "metadata": { "name": "math", - "resourceVersion": "1708548629808", + "resourceVersion": "1708817676338", "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { @@ -40,13 +40,13 @@ "examples": [ { "name": "constant addition", - "queryPayload": { + "saveModel": { "expression": "$A + 10" } }, { "name": "math with two queries", - "queryPayload": { + "saveModel": { "expression": "$A - $B" } } @@ -56,7 +56,7 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1708548629808", + "resourceVersion": "1708817676338", "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { @@ -84,7 +84,7 @@ "last" ], "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", - "x-enum-description": { + "x-enum-dictionary": { "mean": "The mean", "sum": "The sum" } @@ -98,7 +98,7 @@ "replaceNN" ], "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "x-enum-description": { + "x-enum-dictionary": { "dropNN": "Drop non-numbers", "replaceNN": "Replace non-numbers" } @@ -127,7 +127,7 @@ "examples": [ { "name": "get max value", - "queryPayload": { + "saveModel": { "expression": "$A", "reducer": "max", "settings": { @@ -153,31 +153,31 @@ ], "querySchema": { "$schema": "https://json-schema.org/draft-04/schema", + "additionalProperties": false, + "description": "QueryType = resample", "properties": { - "expression": { - "type": "string", - "description": "The math expression" - }, - "window": { - "type": "string", - "description": "A time duration string" - }, "downsampler": { - "type": "string", - "description": "The reducer" + "description": "The reducer", + "type": "string" }, - "upsampler": { - "type": "string", - "description": "The reducer" + "expression": { + "description": "The math expression", + "type": "string" }, "loadedDimensions": { "additionalProperties": true, "type": "object", "x-grafana-type": "data.DataFrame" + }, + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "A time duration string", + "type": "string" } }, - "additionalProperties": false, - "type": "object", "required": [ "expression", "window", @@ -185,7 +185,7 @@ "upsampler", "loadedDimensions" ], - "description": "QueryType = resample" + "type": "object" } } } diff --git a/experimental/spec/example/types_test.go b/experimental/spec/example/query_test.go similarity index 56% rename from experimental/spec/example/types_test.go rename to experimental/spec/example/query_test.go index 6982eb6a4..c18f391d5 100644 --- a/experimental/spec/example/types_test.go +++ b/experimental/spec/example/query_test.go @@ -1,15 +1,17 @@ package example import ( + "encoding/json" + "fmt" "reflect" "testing" - schema "github.com/grafana/grafana-plugin-sdk-go/experimental/spec" + "github.com/grafana/grafana-plugin-sdk-go/experimental/spec" "github.com/stretchr/testify/require" ) func TestQueryTypeDefinitions(t *testing.T) { - builder, err := schema.NewSchemaBuilder(schema.BuilderOptions{ + builder, err := spec.NewSchemaBuilder(spec.BuilderOptions{ BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/spec/example", CodePath: "./", // We need to identify the enum fields explicitly :( @@ -20,31 +22,31 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }) require.NoError(t, err) - err = builder.AddQueries(schema.QueryTypeInfo{ - Discriminators: schema.NewDiscriminators("queryType", QueryTypeMath), + err = builder.AddQueries(spec.QueryTypeInfo{ + Discriminators: spec.NewDiscriminators("queryType", QueryTypeMath), GoType: reflect.TypeOf(&MathQuery{}), - Examples: []schema.QueryExample{ + Examples: []spec.QueryExample{ { Name: "constant addition", - QueryPayload: MathQuery{ + SaveModel: MathQuery{ Expression: "$A + 10", }, }, { Name: "math with two queries", - QueryPayload: MathQuery{ + SaveModel: MathQuery{ Expression: "$A - $B", }, }, }, }, - schema.QueryTypeInfo{ - Discriminators: schema.NewDiscriminators("queryType", QueryTypeReduce), + spec.QueryTypeInfo{ + Discriminators: spec.NewDiscriminators("queryType", QueryTypeReduce), GoType: reflect.TypeOf(&ReduceQuery{}), - Examples: []schema.QueryExample{ + Examples: []spec.QueryExample{ { Name: "get max value", - QueryPayload: ReduceQuery{ + SaveModel: ReduceQuery{ Expression: "$A", Reducer: ReducerMax, Settings: ReduceSettings{ @@ -54,11 +56,19 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }, }, - schema.QueryTypeInfo{ - Discriminators: schema.NewDiscriminators("queryType", QueryTypeResample), + spec.QueryTypeInfo{ + Discriminators: spec.NewDiscriminators("queryType", QueryTypeResample), GoType: reflect.TypeOf(&ResampleQuery{}), }) require.NoError(t, err) - builder.UpdateQueryDefinition(t, "./") + defs := builder.UpdateQueryDefinition(t, "./") + + queries, err := spec.GetExampleQueries(defs) + require.NoError(t, err) + + out, err := json.MarshalIndent(queries, "", " ") + require.NoError(t, err) + + fmt.Printf("%s", string(out)) } diff --git a/experimental/spec/example/testdata/sample_query_request.json b/experimental/spec/example/testdata/sample_query_request.json new file mode 100644 index 000000000..395a65aa9 --- /dev/null +++ b/experimental/spec/example/testdata/sample_query_request.json @@ -0,0 +1,22 @@ +{ + "from": "now-1h", + "to": "now", + "queries": [ + { + "refId": "A", + "expression": "$A + 10" + }, + { + "refId": "B", + "expression": "$A - $B" + }, + { + "refId": "C", + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + ] +} diff --git a/experimental/spec/query.go b/experimental/spec/query.go index 2a1acc21f..de5a32eb5 100644 --- a/experimental/spec/query.go +++ b/experimental/spec/query.go @@ -63,12 +63,7 @@ type QueryExample struct { // Version identifier or empty if only one exists Name string `json:"name,omitempty"` - // An example payload -- this should not require the frontend code to - // pre-process anything - QueryPayload any `json:"queryPayload,omitempty"` - - // An example save model -- this will require frontend code to convert it - // into a valid query payload + // An example value saved that can be saved in a dashboard SaveModel any `json:"saveModel,omitempty"` } @@ -152,6 +147,6 @@ var f embed.FS // Get the cached feature list (exposed as a k8s resource) func GetCommonJSONSchema() json.RawMessage { - body, _ := f.ReadFile("common.jsonschema") + body, _ := f.ReadFile("query.schema.json") return body } diff --git a/experimental/spec/query.schema.json b/experimental/spec/query.schema.json index 3c66d539a..9ed34f630 100644 --- a/experimental/spec/query.schema.json +++ b/experimental/spec/query.schema.json @@ -3,12 +3,14 @@ "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/spec/common-query-properties", "properties": { "refId": { - "type": "string" + "type": "string", + "description": "RefID is the unique identifier of the query, set by the frontend call." }, "resultAssertions": { "properties": { "type": { - "type": "string" + "type": "string", + "description": "Type asserts that the frame matches a known type structure." }, "typeVersion": { "items": { @@ -16,28 +18,34 @@ }, "type": "array", "maxItems": 2, - "minItems": 2 + "minItems": 2, + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." }, "maxBytes": { - "type": "integer" + "type": "integer", + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" }, "maxFrames": { - "type": "integer" + "type": "integer", + "description": "Maximum frame count" } }, "additionalProperties": false, "type": "object", "required": [ "typeVersion" - ] + ], + "description": "Optionally define expected query result behavior" }, "timeRange": { "properties": { "from": { - "type": "string" + "type": "string", + "description": "From is the start time of the query." }, "to": { - "type": "string" + "type": "string", + "description": "To is the end time of the query." } }, "additionalProperties": false, @@ -45,15 +53,18 @@ "required": [ "from", "to" - ] + ], + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" }, "datasource": { "properties": { "type": { - "type": "string" + "type": "string", + "description": "The datasource plugin type" }, "uid": { - "type": "string" + "type": "string", + "description": "Datasource UID" } }, "additionalProperties": false, @@ -61,19 +72,24 @@ "required": [ "type", "uid" - ] + ], + "description": "The datasource" }, "queryType": { - "type": "string" + "type": "string", + "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." }, "maxDataPoints": { - "type": "integer" + "type": "integer", + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" }, "intervalMs": { - "type": "number" + "type": "number", + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" }, "hide": { - "type": "boolean" + "type": "boolean", + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" } }, "additionalProperties": false, diff --git a/experimental/spec/query_parser.go b/experimental/spec/query_parser.go index 54278501b..4503fe55a 100644 --- a/experimental/spec/query_parser.go +++ b/experimental/spec/query_parser.go @@ -1,15 +1,40 @@ package spec import ( + "encoding/json" + "unsafe" + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + j "github.com/json-iterator/go" ) +func init() { //nolint:gochecknoinits + jsoniter.RegisterTypeEncoder("spec.GenericDataQuery", &genericQueryCodec{}) + jsoniter.RegisterTypeDecoder("spec.GenericDataQuery", &genericQueryCodec{}) +} + // GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing type GenericDataQuery struct { CommonQueryProperties `json:",inline"` // Additional Properties (that live at the root) - Additional map[string]any `json:",inline"` + additional map[string]any `json:"-"` // note this uses custom JSON marshalling +} + +type QueryRequest[Q any] struct { + // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. + // example: now-1h + From string `json:"from,omitempty"` + + // To End time in epoch timestamps in milliseconds or relative using Grafana time units. + // example: now + To string `json:"to,omitempty"` + + // Each item has a + Queries []Q `json:"queries"` + + // required: false + Debug bool `json:"debug,omitempty"` } // Generic query parser pattern. @@ -24,10 +49,6 @@ type TypedQueryParser[Q any] interface { ) (Q, error) } -var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) - -type GenericQueryParser struct{} - var commonKeys = map[string]bool{ "refId": true, "resultAssertions": true, @@ -40,13 +61,17 @@ var commonKeys = map[string]bool{ "hide": true, } +var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) + +type GenericQueryParser struct{} + // ParseQuery implements TypedQueryParser. func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator) (GenericDataQuery, error) { - q := GenericDataQuery{CommonQueryProperties: common, Additional: make(map[string]any)} + q := GenericDataQuery{CommonQueryProperties: common, additional: make(map[string]any)} field, err := iter.ReadObject() for field != "" && err == nil { if !commonKeys[field] { - q.Additional[field], err = iter.Read() + q.additional[field], err = iter.Read() if err != nil { return q, err } @@ -55,3 +80,169 @@ func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsonit } return q, err } + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery. +func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { + if g == nil { + return nil + } + out := new(GenericDataQuery) + jj, err := json.Marshal(g) + if err != nil { + _ = json.Unmarshal(jj, out) + } + return out +} + +func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { + clone := g.DeepCopy() + *out = *clone +} + +type genericQueryCodec struct{} + +func (codec *genericQueryCodec) IsEmpty(ptr unsafe.Pointer) bool { + return false +} + +func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { + q := (*GenericDataQuery)(ptr) + writeQuery(q, stream) +} + +func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { + q := GenericDataQuery{} + err := readQuery(&q, jsoniter.NewIterator(iter)) + if err != nil { + // keep existing iter error if it exists + if iter.Error == nil { + iter.Error = err + } + return + } + *((*GenericDataQuery)(ptr)) = q +} + +// MarshalJSON writes JSON including the common and custom values +func (g GenericDataQuery) MarshalJSON() ([]byte, error) { + cfg := j.ConfigCompatibleWithStandardLibrary + stream := cfg.BorrowStream(nil) + defer cfg.ReturnStream(stream) + + writeQuery(&g, stream) + return append([]byte(nil), stream.Buffer()...), stream.Error +} + +// UnmarshalJSON reads a query from json byte array +func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { + iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, b) + if err != nil { + return err + } + return readQuery(g, iter) +} + +func writeQuery(g *GenericDataQuery, stream *j.Stream) { + q := g.CommonQueryProperties + stream.WriteObjectStart() + stream.WriteObjectField("refId") + stream.WriteVal(g.RefID) + + if q.ResultAssertions != nil { + stream.WriteMore() + stream.WriteObjectField("resultAssertions") + stream.WriteVal(g.ResultAssertions) + } + + if q.TimeRange != nil { + stream.WriteMore() + stream.WriteObjectField("timeRange") + stream.WriteVal(g.TimeRange) + } + + if q.Datasource != nil { + stream.WriteMore() + stream.WriteObjectField("datasource") + stream.WriteVal(g.Datasource) + } + + if q.DatasourceID > 0 { + stream.WriteMore() + stream.WriteObjectField("datasourceId") + stream.WriteVal(g.DatasourceID) + } + + if q.QueryType != "" { + stream.WriteMore() + stream.WriteObjectField("queryType") + stream.WriteVal(g.QueryType) + } + + if q.MaxDataPoints > 0 { + stream.WriteMore() + stream.WriteObjectField("maxDataPoints") + stream.WriteVal(g.MaxDataPoints) + } + + if q.IntervalMS > 0 { + stream.WriteMore() + stream.WriteObjectField("intervalMs") + stream.WriteVal(g.IntervalMS) + } + + if q.Hide { + stream.WriteMore() + stream.WriteObjectField("hide") + stream.WriteVal(g.Hide) + } + + // The additional properties + if g.additional != nil { + for k, v := range g.additional { + stream.WriteMore() + stream.WriteObjectField(k) + stream.WriteVal(v) + } + } + stream.WriteObjectEnd() +} + +func readQuery(g *GenericDataQuery, iter *jsoniter.Iterator) error { + var err error + field := "" + for field, err = iter.ReadObject(); field != ""; field, err = iter.ReadObject() { + switch field { + case "refId": + g.RefID, err = iter.ReadString() + case "resultAssertions": + err = iter.ReadVal(&g.ResultAssertions) + case "timeRange": + err = iter.ReadVal(&g.TimeRange) + case "datasource": + err = iter.ReadVal(&g.Datasource) + case "datasourceId": + g.DatasourceID, err = iter.ReadInt64() + case "queryType": + g.QueryType, err = iter.ReadString() + case "maxDataPoints": + g.MaxDataPoints, err = iter.ReadInt64() + case "intervalMs": + g.IntervalMS, err = iter.ReadFloat64() + case "hide": + g.Hide, err = iter.ReadBool() + default: + v, err := iter.Read() + if err != nil { + return err + } + if g.additional == nil { + g.additional = make(map[string]any) + } + g.additional[field] = v + } + if err != nil { + return err + } + } + return err +} diff --git a/experimental/spec/query_test.go b/experimental/spec/query_test.go index e90726d43..5fdbec661 100644 --- a/experimental/spec/query_test.go +++ b/experimental/spec/query_test.go @@ -14,7 +14,7 @@ import ( func TestCommonSupport(t *testing.T) { r := new(jsonschema.Reflector) r.DoNotReference = true - err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/query", "./") + err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/spec", "./") require.NoError(t, err) query := r.Reflect(&CommonQueryProperties{}) From e8544028e5d95ec8ee5c62c435244a924d4b11f4 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sat, 24 Feb 2024 15:55:16 -0800 Subject: [PATCH 18/71] more reference stuff --- experimental/spec/builder.go | 2 +- .../spec/example/query.post.schema.json | 86 +++++++++---------- .../spec/example/query.save.schema.json | 30 +++---- .../testdata/sample_query_request.json | 20 +---- 4 files changed, 61 insertions(+), 77 deletions(-) diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go index b06505bc4..b3180a544 100644 --- a/experimental/spec/builder.go +++ b/experimental/spec/builder.go @@ -388,7 +388,7 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav continue // } definitions[pair.Key] = pair.Value - common[pair.Key] = &jsonschema.Schema{Ref: "#/definitions/" + pair.Key} + common[pair.Key] = &jsonschema.Schema{Ref: "#/defs/" + pair.Key} } // The types for each query type diff --git a/experimental/spec/example/query.post.schema.json b/experimental/spec/example/query.post.schema.json index 8ff42e65f..1bbc24dcf 100644 --- a/experimental/spec/example/query.post.schema.json +++ b/experimental/spec/example/query.post.schema.json @@ -94,29 +94,29 @@ "type": "string", "pattern": "^math$" }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" + "timeRange": { + "$ref": "#/defs/timeRange" }, - "hide": { - "$ref": "#/definitions/hide" + "maxDataPoints": { + "$ref": "#/defs/maxDataPoints" }, - "datasource": { - "$ref": "#/definitions/datasource" + "hide": { + "$ref": "#/defs/hide" }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "refId": { + "$ref": "#/defs/refId" }, - "timeRange": { - "$ref": "#/definitions/timeRange" + "datasource": { + "$ref": "#/defs/datasource" }, "datasourceId": { - "$ref": "#/definitions/datasourceId" + "$ref": "#/defs/datasourceId" }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" + "intervalMs": { + "$ref": "#/defs/intervalMs" }, - "refId": { - "$ref": "#/definitions/refId" + "resultAssertions": { + "$ref": "#/defs/resultAssertions" } }, "additionalProperties": false, @@ -171,29 +171,29 @@ "type": "string", "pattern": "^reduce$" }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" + "maxDataPoints": { + "$ref": "#/defs/maxDataPoints" }, "hide": { - "$ref": "#/definitions/hide" - }, - "datasource": { - "$ref": "#/definitions/datasource" + "$ref": "#/defs/hide" }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "refId": { + "$ref": "#/defs/refId" }, "timeRange": { - "$ref": "#/definitions/timeRange" + "$ref": "#/defs/timeRange" }, "datasourceId": { - "$ref": "#/definitions/datasourceId" + "$ref": "#/defs/datasourceId" }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" + "intervalMs": { + "$ref": "#/defs/intervalMs" }, - "refId": { - "$ref": "#/definitions/refId" + "resultAssertions": { + "$ref": "#/defs/resultAssertions" + }, + "datasource": { + "$ref": "#/defs/datasource" } }, "additionalProperties": false, @@ -232,29 +232,29 @@ "type": "string", "pattern": "^resample$" }, - "refId": { - "$ref": "#/definitions/refId" - }, "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "$ref": "#/defs/resultAssertions" }, - "timeRange": { - "$ref": "#/definitions/timeRange" + "datasource": { + "$ref": "#/defs/datasource" }, "datasourceId": { - "$ref": "#/definitions/datasourceId" + "$ref": "#/defs/datasourceId" }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" + "intervalMs": { + "$ref": "#/defs/intervalMs" }, - "datasource": { - "$ref": "#/definitions/datasource" + "refId": { + "$ref": "#/defs/refId" }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" + "timeRange": { + "$ref": "#/defs/timeRange" + }, + "maxDataPoints": { + "$ref": "#/defs/maxDataPoints" }, "hide": { - "$ref": "#/definitions/hide" + "$ref": "#/defs/hide" } }, "additionalProperties": false, diff --git a/experimental/spec/example/query.save.schema.json b/experimental/spec/example/query.save.schema.json index 816f64f1d..cead43158 100644 --- a/experimental/spec/example/query.save.schema.json +++ b/experimental/spec/example/query.save.schema.json @@ -73,19 +73,19 @@ "pattern": "^math$" }, "hide": { - "$ref": "#/definitions/hide" + "$ref": "#/defs/hide" }, "refId": { - "$ref": "#/definitions/refId" + "$ref": "#/defs/refId" }, "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "$ref": "#/defs/resultAssertions" }, "datasource": { - "$ref": "#/definitions/datasource" + "$ref": "#/defs/datasource" }, "datasourceId": { - "$ref": "#/definitions/datasourceId" + "$ref": "#/defs/datasourceId" } }, "additionalProperties": false, @@ -141,19 +141,19 @@ "pattern": "^reduce$" }, "refId": { - "$ref": "#/definitions/refId" + "$ref": "#/defs/refId" }, "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "$ref": "#/defs/resultAssertions" }, "datasource": { - "$ref": "#/definitions/datasource" + "$ref": "#/defs/datasource" }, "datasourceId": { - "$ref": "#/definitions/datasourceId" + "$ref": "#/defs/datasourceId" }, "hide": { - "$ref": "#/definitions/hide" + "$ref": "#/defs/hide" } }, "additionalProperties": false, @@ -193,19 +193,19 @@ "pattern": "^resample$" }, "datasource": { - "$ref": "#/definitions/datasource" + "$ref": "#/defs/datasource" }, "datasourceId": { - "$ref": "#/definitions/datasourceId" + "$ref": "#/defs/datasourceId" }, "hide": { - "$ref": "#/definitions/hide" + "$ref": "#/defs/hide" }, "refId": { - "$ref": "#/definitions/refId" + "$ref": "#/defs/refId" }, "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "$ref": "#/defs/resultAssertions" } }, "additionalProperties": false, diff --git a/experimental/spec/example/testdata/sample_query_request.json b/experimental/spec/example/testdata/sample_query_request.json index 395a65aa9..1669f11c5 100644 --- a/experimental/spec/example/testdata/sample_query_request.json +++ b/experimental/spec/example/testdata/sample_query_request.json @@ -1,22 +1,6 @@ { - "from": "now-1h", - "to": "now", - "queries": [ - { + "$schema": "https://raw.githubusercontent.com/grafana/grafana-plugin-sdk-go/10b145dd3c45ac25f8048358164099ef6d747687/experimental/spec/example/query.post.schema.json", "refId": "A", "expression": "$A + 10" - }, - { - "refId": "B", - "expression": "$A - $B" - }, - { - "refId": "C", - "expression": "$A", - "reducer": "max", - "settings": { - "mode": "dropNN" - } - } - ] + } From 074b8f808a697d306ce92a29c8d9820bb2981d10 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sat, 24 Feb 2024 16:46:56 -0800 Subject: [PATCH 19/71] more reference stuff --- experimental/spec/builder.go | 25 +++- .../spec/example/query.post.schema.json | 134 +++++++++--------- .../spec/example/query.save.schema.json | 109 ++++++++------ .../testdata/sample_query_request.json | 31 +++- experimental/spec/query_parser.go | 74 ++++++++++ 5 files changed, 257 insertions(+), 116 deletions(-) diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go index b3180a544..929eabcc3 100644 --- a/experimental/spec/builder.go +++ b/experimental/spec/builder.go @@ -287,12 +287,9 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) QueryTypeDe require.NoError(t, err) // Read query info - r := new(jsonschema.Reflector) - r.DoNotReference = true - err = r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/spec", "./") + query := &jsonschema.Schema{} + err = query.UnmarshalJSON(GetCommonJSONSchema()) require.NoError(t, err) - - query := r.Reflect(&CommonQueryProperties{}) query.Version = draft04 // used by kube-openapi query.Description = "Query properties shared by all data sources" @@ -380,7 +377,7 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav descr = "Save model (the payload saved in dashboards and alerts)" } - ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true, "timeRange": true} + ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true} definitions := make(jsonschema.Definitions) common := make(map[string]*jsonschema.Schema) for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { @@ -388,7 +385,7 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav continue // } definitions[pair.Key] = pair.Value - common[pair.Key] = &jsonschema.Schema{Ref: "#/defs/" + pair.Key} + common[pair.Key] = &jsonschema.Schema{Ref: "#/$defs/" + pair.Key} } // The types for each query type @@ -507,8 +504,11 @@ func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { func GetExampleQueries(defs QueryTypeDefinitionList) (QueryRequest[GenericDataQuery], error) { rsp := QueryRequest[GenericDataQuery]{ + From: "now-1h", + To: "now", Queries: []GenericDataQuery{}, } + for _, def := range defs.Items { for _, sample := range def.Spec.Examples { if sample.SaveModel != nil { @@ -518,6 +518,17 @@ func GetExampleQueries(defs QueryTypeDefinitionList) (QueryRequest[GenericDataQu sample.Name, def.ObjectMeta.Name, err) } q.RefID = string(rune('A' + len(rsp.Queries))) + for _, dis := range def.Spec.Discriminators { + _ = q.Set(dis.Field, dis.Value) + } + + if q.MaxDataPoints < 1 { + q.MaxDataPoints = 1000 + } + if q.IntervalMS < 1 { + q.IntervalMS = 5 + } + rsp.Queries = append(rsp.Queries, *q) } } diff --git a/experimental/spec/example/query.post.schema.json b/experimental/spec/example/query.post.schema.json index 1bbc24dcf..1fd8d2b10 100644 --- a/experimental/spec/example/query.post.schema.json +++ b/experimental/spec/example/query.post.schema.json @@ -4,10 +4,12 @@ "datasource": { "properties": { "type": { - "type": "string" + "type": "string", + "description": "The datasource plugin type" }, "uid": { - "type": "string" + "type": "string", + "description": "Datasource UID" } }, "additionalProperties": false, @@ -15,30 +17,34 @@ "required": [ "type", "uid" - ] - }, - "datasourceId": { - "type": "integer" + ], + "description": "The datasource" }, "hide": { - "type": "boolean" + "type": "boolean", + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" }, "intervalMs": { - "type": "number" + "type": "number", + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" }, "maxDataPoints": { - "type": "integer" + "type": "integer", + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" }, "queryType": { - "type": "string" + "type": "string", + "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." }, "refId": { - "type": "string" + "type": "string", + "description": "RefID is the unique identifier of the query, set by the frontend call." }, "resultAssertions": { "properties": { "type": { - "type": "string" + "type": "string", + "description": "Type asserts that the frame matches a known type structure." }, "typeVersion": { "items": { @@ -46,28 +52,34 @@ }, "type": "array", "maxItems": 2, - "minItems": 2 + "minItems": 2, + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." }, "maxBytes": { - "type": "integer" + "type": "integer", + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" }, "maxFrames": { - "type": "integer" + "type": "integer", + "description": "Maximum frame count" } }, "additionalProperties": false, "type": "object", "required": [ "typeVersion" - ] + ], + "description": "Optionally define expected query result behavior" }, "timeRange": { "properties": { "from": { - "type": "string" + "type": "string", + "description": "From is the start time of the query." }, "to": { - "type": "string" + "type": "string", + "description": "To is the end time of the query." } }, "additionalProperties": false, @@ -75,7 +87,8 @@ "required": [ "from", "to" - ] + ], + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" } }, "oneOf": [ @@ -95,28 +108,25 @@ "pattern": "^math$" }, "timeRange": { - "$ref": "#/defs/timeRange" + "$ref": "#/$defs/timeRange" + }, + "datasource": { + "$ref": "#/$defs/datasource" }, "maxDataPoints": { - "$ref": "#/defs/maxDataPoints" + "$ref": "#/$defs/maxDataPoints" + }, + "intervalMs": { + "$ref": "#/$defs/intervalMs" }, "hide": { - "$ref": "#/defs/hide" + "$ref": "#/$defs/hide" }, "refId": { - "$ref": "#/defs/refId" - }, - "datasource": { - "$ref": "#/defs/datasource" - }, - "datasourceId": { - "$ref": "#/defs/datasourceId" - }, - "intervalMs": { - "$ref": "#/defs/intervalMs" + "$ref": "#/$defs/refId" }, "resultAssertions": { - "$ref": "#/defs/resultAssertions" + "$ref": "#/$defs/resultAssertions" } }, "additionalProperties": false, @@ -171,29 +181,26 @@ "type": "string", "pattern": "^reduce$" }, - "maxDataPoints": { - "$ref": "#/defs/maxDataPoints" + "timeRange": { + "$ref": "#/$defs/timeRange" }, - "hide": { - "$ref": "#/defs/hide" + "datasource": { + "$ref": "#/$defs/datasource" }, - "refId": { - "$ref": "#/defs/refId" + "maxDataPoints": { + "$ref": "#/$defs/maxDataPoints" }, - "timeRange": { - "$ref": "#/defs/timeRange" + "intervalMs": { + "$ref": "#/$defs/intervalMs" }, - "datasourceId": { - "$ref": "#/defs/datasourceId" + "hide": { + "$ref": "#/$defs/hide" }, - "intervalMs": { - "$ref": "#/defs/intervalMs" + "refId": { + "$ref": "#/$defs/refId" }, "resultAssertions": { - "$ref": "#/defs/resultAssertions" - }, - "datasource": { - "$ref": "#/defs/datasource" + "$ref": "#/$defs/resultAssertions" } }, "additionalProperties": false, @@ -232,29 +239,26 @@ "type": "string", "pattern": "^resample$" }, - "resultAssertions": { - "$ref": "#/defs/resultAssertions" - }, - "datasource": { - "$ref": "#/defs/datasource" - }, - "datasourceId": { - "$ref": "#/defs/datasourceId" - }, "intervalMs": { - "$ref": "#/defs/intervalMs" + "$ref": "#/$defs/intervalMs" + }, + "hide": { + "$ref": "#/$defs/hide" }, "refId": { - "$ref": "#/defs/refId" + "$ref": "#/$defs/refId" + }, + "resultAssertions": { + "$ref": "#/$defs/resultAssertions" }, "timeRange": { - "$ref": "#/defs/timeRange" + "$ref": "#/$defs/timeRange" }, - "maxDataPoints": { - "$ref": "#/defs/maxDataPoints" + "datasource": { + "$ref": "#/$defs/datasource" }, - "hide": { - "$ref": "#/defs/hide" + "maxDataPoints": { + "$ref": "#/$defs/maxDataPoints" } }, "additionalProperties": false, diff --git a/experimental/spec/example/query.save.schema.json b/experimental/spec/example/query.save.schema.json index cead43158..019d36e74 100644 --- a/experimental/spec/example/query.save.schema.json +++ b/experimental/spec/example/query.save.schema.json @@ -4,10 +4,12 @@ "datasource": { "properties": { "type": { - "type": "string" + "type": "string", + "description": "The datasource plugin type" }, "uid": { - "type": "string" + "type": "string", + "description": "Datasource UID" } }, "additionalProperties": false, @@ -15,24 +17,26 @@ "required": [ "type", "uid" - ] - }, - "datasourceId": { - "type": "integer" + ], + "description": "The datasource" }, "hide": { - "type": "boolean" + "type": "boolean", + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" }, "queryType": { - "type": "string" + "type": "string", + "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." }, "refId": { - "type": "string" + "type": "string", + "description": "RefID is the unique identifier of the query, set by the frontend call." }, "resultAssertions": { "properties": { "type": { - "type": "string" + "type": "string", + "description": "Type asserts that the frame matches a known type structure." }, "typeVersion": { "items": { @@ -40,20 +44,43 @@ }, "type": "array", "maxItems": 2, - "minItems": 2 + "minItems": 2, + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." }, "maxBytes": { - "type": "integer" + "type": "integer", + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" }, "maxFrames": { - "type": "integer" + "type": "integer", + "description": "Maximum frame count" } }, "additionalProperties": false, "type": "object", "required": [ "typeVersion" - ] + ], + "description": "Optionally define expected query result behavior" + }, + "timeRange": { + "properties": { + "from": { + "type": "string", + "description": "From is the start time of the query." + }, + "to": { + "type": "string", + "description": "To is the end time of the query." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "from", + "to" + ], + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" } }, "oneOf": [ @@ -72,20 +99,20 @@ "type": "string", "pattern": "^math$" }, + "datasource": { + "$ref": "#/$defs/datasource" + }, "hide": { - "$ref": "#/defs/hide" + "$ref": "#/$defs/hide" }, "refId": { - "$ref": "#/defs/refId" + "$ref": "#/$defs/refId" }, "resultAssertions": { - "$ref": "#/defs/resultAssertions" + "$ref": "#/$defs/resultAssertions" }, - "datasource": { - "$ref": "#/defs/datasource" - }, - "datasourceId": { - "$ref": "#/defs/datasourceId" + "timeRange": { + "$ref": "#/$defs/timeRange" } }, "additionalProperties": false, @@ -140,20 +167,20 @@ "type": "string", "pattern": "^reduce$" }, - "refId": { - "$ref": "#/defs/refId" - }, - "resultAssertions": { - "$ref": "#/defs/resultAssertions" + "timeRange": { + "$ref": "#/$defs/timeRange" }, "datasource": { - "$ref": "#/defs/datasource" - }, - "datasourceId": { - "$ref": "#/defs/datasourceId" + "$ref": "#/$defs/datasource" }, "hide": { - "$ref": "#/defs/hide" + "$ref": "#/$defs/hide" + }, + "refId": { + "$ref": "#/$defs/refId" + }, + "resultAssertions": { + "$ref": "#/$defs/resultAssertions" } }, "additionalProperties": false, @@ -192,20 +219,20 @@ "type": "string", "pattern": "^resample$" }, - "datasource": { - "$ref": "#/defs/datasource" - }, - "datasourceId": { - "$ref": "#/defs/datasourceId" - }, "hide": { - "$ref": "#/defs/hide" + "$ref": "#/$defs/hide" }, "refId": { - "$ref": "#/defs/refId" + "$ref": "#/$defs/refId" }, "resultAssertions": { - "$ref": "#/defs/resultAssertions" + "$ref": "#/$defs/resultAssertions" + }, + "timeRange": { + "$ref": "#/$defs/timeRange" + }, + "datasource": { + "$ref": "#/$defs/datasource" } }, "additionalProperties": false, diff --git a/experimental/spec/example/testdata/sample_query_request.json b/experimental/spec/example/testdata/sample_query_request.json index 1669f11c5..871d54dc8 100644 --- a/experimental/spec/example/testdata/sample_query_request.json +++ b/experimental/spec/example/testdata/sample_query_request.json @@ -1,6 +1,31 @@ { - "$schema": "https://raw.githubusercontent.com/grafana/grafana-plugin-sdk-go/10b145dd3c45ac25f8048358164099ef6d747687/experimental/spec/example/query.post.schema.json", + "from": "now-1h", + "to": "now", + "queries": [ + { "refId": "A", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, "expression": "$A + 10" - -} + }, + { + "refId": "B", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A - $B" + }, + { + "refId": "C", + "queryType": "reduce", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + ] +} \ No newline at end of file diff --git a/experimental/spec/query_parser.go b/experimental/spec/query_parser.go index 4503fe55a..abfe86bb7 100644 --- a/experimental/spec/query_parser.go +++ b/experimental/spec/query_parser.go @@ -4,6 +4,7 @@ import ( "encoding/json" "unsafe" + "github.com/grafana/grafana-plugin-sdk-go/data/converters" "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" j "github.com/json-iterator/go" ) @@ -99,6 +100,79 @@ func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { *out = *clone } +// Set allows setting values using key/value pairs +func (g *GenericDataQuery) Set(key string, val any) *GenericDataQuery { + switch key { + case "refId": + g.RefID, _ = val.(string) + case "resultAssertions": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.ResultAssertions) + } + case "timeRange": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.TimeRange) + } + case "datasource": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.Datasource) + } + case "datasourceId": + v, err := converters.JSONValueToInt64.Converter(val) + if err != nil { + g.DatasourceID, _ = v.(int64) + } + case "queryType": + g.QueryType, _ = val.(string) + case "maxDataPoints": + v, err := converters.JSONValueToInt64.Converter(val) + if err != nil { + g.MaxDataPoints, _ = v.(int64) + } + case "intervalMs": + v, err := converters.JSONValueToFloat64.Converter(val) + if err != nil { + g.IntervalMS, _ = v.(float64) + } + case "hide": + g.Hide, _ = val.(bool) + default: + if g.additional == nil { + g.additional = make(map[string]any) + } + g.additional[key] = val + } + return g +} + +func (g *GenericDataQuery) Get(key string) (any, bool) { + switch key { + case "refId": + return g.RefID, true + case "resultAssertions": + return g.ResultAssertions, true + case "timeRange": + return g.TimeRange, true + case "datasource": + return g.Datasource, true + case "datasourceId": + return g.DatasourceID, true + case "queryType": + return g.QueryType, true + case "maxDataPoints": + return g.MaxDataPoints, true + case "intervalMs": + return g.IntervalMS, true + case "hide": + return g.Hide, true + } + v, ok := g.additional[key] + return v, ok +} + type genericQueryCodec struct{} func (codec *genericQueryCodec) IsEmpty(ptr unsafe.Pointer) bool { From c67324318b0b055e73b038cb5eaf3090dc8b953c Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sat, 24 Feb 2024 21:15:11 -0800 Subject: [PATCH 20/71] with definitions --- experimental/spec/builder.go | 17 ++++-- .../spec/example/query.post.schema.json | 60 +++++++++---------- .../spec/example/query.save.schema.json | 52 ++++++++-------- .../testdata/sample_query_request.json | 26 +------- 4 files changed, 70 insertions(+), 85 deletions(-) diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go index 929eabcc3..aaebe8e06 100644 --- a/experimental/spec/builder.go +++ b/experimental/spec/builder.go @@ -371,7 +371,9 @@ func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) Setting } // Converts a set of queries into a single real schema (merged with the common properties) -func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, saveModel bool) (*jsonschema.Schema, error) { +// This returns a the raw bytes because `invopop/jsonschema` requires extra manipulation +// so that the results are readable by `kubernetes/kube-openapi` +func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, saveModel bool) (json.RawMessage, error) { descr := "Query model (the payload sent to /ds/query)" if saveModel { descr = "Save model (the payload saved in dashboards and alerts)" @@ -385,7 +387,7 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav continue // } definitions[pair.Key] = pair.Value - common[pair.Key] = &jsonschema.Schema{Ref: "#/$defs/" + pair.Key} + common[pair.Key] = &jsonschema.Schema{Ref: "#/definitions/" + pair.Key} } // The types for each query type @@ -419,7 +421,6 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav node := queryTypes[0] node.Version = draft04 node.Description = descr - node.Definitions = definitions for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { _, found := node.Properties.Get(pair.Key) if found { @@ -427,7 +428,7 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav } node.Properties.Set(pair.Key, pair.Value) } - return node, nil + return json.MarshalIndent(node, "", " ") } s := &jsonschema.Schema{ @@ -451,7 +452,13 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav s.OneOf = append(s.OneOf, qt) } - return s, nil + + body, err := json.MarshalIndent(s, "", " ") + if err == nil { + v := strings.Replace(string(body), `"$defs": {`, `"definitions": {`, 1) + return []byte(v), nil + } + return body, err } func asJSONSchema(v any) (*jsonschema.Schema, error) { diff --git a/experimental/spec/example/query.post.schema.json b/experimental/spec/example/query.post.schema.json index 1fd8d2b10..d4a204b56 100644 --- a/experimental/spec/example/query.post.schema.json +++ b/experimental/spec/example/query.post.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft-04/schema", - "$defs": { + "definitions": { "datasource": { "properties": { "type": { @@ -107,26 +107,26 @@ "type": "string", "pattern": "^math$" }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, "timeRange": { - "$ref": "#/$defs/timeRange" + "$ref": "#/definitions/timeRange" }, "datasource": { - "$ref": "#/$defs/datasource" + "$ref": "#/definitions/datasource" }, "maxDataPoints": { - "$ref": "#/$defs/maxDataPoints" + "$ref": "#/definitions/maxDataPoints" }, "intervalMs": { - "$ref": "#/$defs/intervalMs" + "$ref": "#/definitions/intervalMs" }, "hide": { - "$ref": "#/$defs/hide" + "$ref": "#/definitions/hide" }, "refId": { - "$ref": "#/$defs/refId" - }, - "resultAssertions": { - "$ref": "#/$defs/resultAssertions" + "$ref": "#/definitions/refId" } }, "additionalProperties": false, @@ -181,26 +181,26 @@ "type": "string", "pattern": "^reduce$" }, - "timeRange": { - "$ref": "#/$defs/timeRange" - }, - "datasource": { - "$ref": "#/$defs/datasource" - }, "maxDataPoints": { - "$ref": "#/$defs/maxDataPoints" + "$ref": "#/definitions/maxDataPoints" }, "intervalMs": { - "$ref": "#/$defs/intervalMs" + "$ref": "#/definitions/intervalMs" }, "hide": { - "$ref": "#/$defs/hide" + "$ref": "#/definitions/hide" }, "refId": { - "$ref": "#/$defs/refId" + "$ref": "#/definitions/refId" }, "resultAssertions": { - "$ref": "#/$defs/resultAssertions" + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" } }, "additionalProperties": false, @@ -239,26 +239,26 @@ "type": "string", "pattern": "^resample$" }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + }, "intervalMs": { - "$ref": "#/$defs/intervalMs" + "$ref": "#/definitions/intervalMs" }, "hide": { - "$ref": "#/$defs/hide" + "$ref": "#/definitions/hide" }, "refId": { - "$ref": "#/$defs/refId" + "$ref": "#/definitions/refId" }, "resultAssertions": { - "$ref": "#/$defs/resultAssertions" + "$ref": "#/definitions/resultAssertions" }, "timeRange": { - "$ref": "#/$defs/timeRange" + "$ref": "#/definitions/timeRange" }, "datasource": { - "$ref": "#/$defs/datasource" - }, - "maxDataPoints": { - "$ref": "#/$defs/maxDataPoints" + "$ref": "#/definitions/datasource" } }, "additionalProperties": false, diff --git a/experimental/spec/example/query.save.schema.json b/experimental/spec/example/query.save.schema.json index 019d36e74..ac2ab87c0 100644 --- a/experimental/spec/example/query.save.schema.json +++ b/experimental/spec/example/query.save.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft-04/schema", - "$defs": { + "definitions": { "datasource": { "properties": { "type": { @@ -99,20 +99,20 @@ "type": "string", "pattern": "^math$" }, - "datasource": { - "$ref": "#/$defs/datasource" - }, - "hide": { - "$ref": "#/$defs/hide" - }, "refId": { - "$ref": "#/$defs/refId" + "$ref": "#/definitions/refId" }, "resultAssertions": { - "$ref": "#/$defs/resultAssertions" + "$ref": "#/definitions/resultAssertions" }, "timeRange": { - "$ref": "#/$defs/timeRange" + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "hide": { + "$ref": "#/definitions/hide" } }, "additionalProperties": false, @@ -167,20 +167,20 @@ "type": "string", "pattern": "^reduce$" }, - "timeRange": { - "$ref": "#/$defs/timeRange" - }, - "datasource": { - "$ref": "#/$defs/datasource" - }, "hide": { - "$ref": "#/$defs/hide" + "$ref": "#/definitions/hide" }, "refId": { - "$ref": "#/$defs/refId" + "$ref": "#/definitions/refId" }, "resultAssertions": { - "$ref": "#/$defs/resultAssertions" + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" } }, "additionalProperties": false, @@ -219,20 +219,20 @@ "type": "string", "pattern": "^resample$" }, - "hide": { - "$ref": "#/$defs/hide" - }, "refId": { - "$ref": "#/$defs/refId" + "$ref": "#/definitions/refId" }, "resultAssertions": { - "$ref": "#/$defs/resultAssertions" + "$ref": "#/definitions/resultAssertions" }, "timeRange": { - "$ref": "#/$defs/timeRange" + "$ref": "#/definitions/timeRange" }, "datasource": { - "$ref": "#/$defs/datasource" + "$ref": "#/definitions/datasource" + }, + "hide": { + "$ref": "#/definitions/hide" } }, "additionalProperties": false, diff --git a/experimental/spec/example/testdata/sample_query_request.json b/experimental/spec/example/testdata/sample_query_request.json index 871d54dc8..d79ea531d 100644 --- a/experimental/spec/example/testdata/sample_query_request.json +++ b/experimental/spec/example/testdata/sample_query_request.json @@ -1,31 +1,9 @@ { - "from": "now-1h", - "to": "now", - "queries": [ - { + "$schema": "file:/Users/ryan/workspace/grafana/more/grafana-plugin-sdk-go/experimental/spec/example/query.post.schema.json", + "refId": "A", "queryType": "math", "maxDataPoints": 1000, "intervalMs": 5, "expression": "$A + 10" - }, - { - "refId": "B", - "queryType": "math", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A - $B" - }, - { - "refId": "C", - "queryType": "reduce", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A", - "reducer": "max", - "settings": { - "mode": "dropNN" - } - } - ] } \ No newline at end of file From 940bc8e6d0c89603268d95ee0041514bcc608f90 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sat, 24 Feb 2024 21:21:05 -0800 Subject: [PATCH 21/71] with definitions --- experimental/spec/example/query.save.schema.json | 6 +++++- .../spec/example/testdata/sample_query_request.json | 13 ++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/experimental/spec/example/query.save.schema.json b/experimental/spec/example/query.save.schema.json index ac2ab87c0..56d018526 100644 --- a/experimental/spec/example/query.save.schema.json +++ b/experimental/spec/example/query.save.schema.json @@ -249,7 +249,11 @@ "description": "QueryType = resample" } ], - "properties": {}, + "properties": { + "$schema": { + "type": "string" + } + }, "type": "object", "description": "Save model (the payload saved in dashboards and alerts)" } \ No newline at end of file diff --git a/experimental/spec/example/testdata/sample_query_request.json b/experimental/spec/example/testdata/sample_query_request.json index d79ea531d..1ef064a3c 100644 --- a/experimental/spec/example/testdata/sample_query_request.json +++ b/experimental/spec/example/testdata/sample_query_request.json @@ -1,9 +1,8 @@ { - "$schema": "file:/Users/ryan/workspace/grafana/more/grafana-plugin-sdk-go/experimental/spec/example/query.post.schema.json", - - "refId": "A", - "queryType": "math", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A + 10" + "$schema": "https://raw.githubusercontent.com/grafana/grafana-plugin-sdk-go/c67324318b0b055e73b038cb5eaf3090dc8b953c/experimental/spec/example/query.post.schema.json", + "refId": "A", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A + 10" } \ No newline at end of file From 5c2de2719966cf168a719a5a1ddbdcc479f23b57 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sat, 24 Feb 2024 22:06:35 -0800 Subject: [PATCH 22/71] cleanup --- experimental/spec/builder.go | 104 ++++-- .../spec/example/query.post.schema.json | 281 ---------------- .../spec/example/query.request.schema.json | 311 ++++++++++++++++++ ...ery.save.schema.json => query.schema.json} | 14 +- .../testdata/sample_query_request.json | 37 ++- 5 files changed, 419 insertions(+), 328 deletions(-) delete mode 100644 experimental/spec/example/query.post.schema.json create mode 100644 experimental/spec/example/query.request.schema.json rename experimental/spec/example/{query.save.schema.json => query.schema.json} (97%) diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go index aaebe8e06..a2e54e27e 100644 --- a/experimental/spec/builder.go +++ b/experimental/spec/builder.go @@ -295,8 +295,8 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) QueryTypeDe // Now update the query files //---------------------------- - outfile = filepath.Join(outdir, "query.post.schema.json") - schema, err := toQuerySchema(query, defs, false) + outfile = filepath.Join(outdir, "query.request.schema.json") + schema, err := toQuerySchema(query, defs, true) require.NoError(t, err) body, _ = os.ReadFile(outfile) @@ -304,8 +304,8 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) QueryTypeDe // Now update the query files //---------------------------- - outfile = filepath.Join(outdir, "query.save.schema.json") - schema, err = toQuerySchema(query, defs, true) + outfile = filepath.Join(outdir, "query.schema.json") + schema, err = toQuerySchema(query, defs, false) require.NoError(t, err) body, _ = os.ReadFile(outfile) @@ -373,17 +373,17 @@ func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) Setting // Converts a set of queries into a single real schema (merged with the common properties) // This returns a the raw bytes because `invopop/jsonschema` requires extra manipulation // so that the results are readable by `kubernetes/kube-openapi` -func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, saveModel bool) (json.RawMessage, error) { - descr := "Query model (the payload sent to /ds/query)" - if saveModel { - descr = "Save model (the payload saved in dashboards and alerts)" +func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, isRequest bool) (json.RawMessage, error) { + descr := "object saved in dashboard/alert" + if isRequest { + descr = "Datasource request model" } ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true} definitions := make(jsonschema.Definitions) common := make(map[string]*jsonschema.Schema) for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { - if saveModel && ignoreForSave[pair.Key] { + if !isRequest && ignoreForSave[pair.Key] { continue // } definitions[pair.Key] = pair.Value @@ -416,21 +416,6 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav queryTypes = append(queryTypes, node) } - // Single node -- just union the global and local properties - if len(queryTypes) == 1 { - node := queryTypes[0] - node.Version = draft04 - node.Description = descr - for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { - _, found := node.Properties.Get(pair.Key) - if found { - continue - } - node.Properties.Set(pair.Key, pair.Value) - } - return json.MarshalIndent(node, "", " ") - } - s := &jsonschema.Schema{ Type: "object", Version: draft04, @@ -439,28 +424,85 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, sav Description: descr, } - for _, qt := range queryTypes { - qt.Required = append(qt.Required, "refId") - - for k, v := range common { - _, found := qt.Properties.Get(k) + // Single node -- just union the global and local properties + if len(queryTypes) == 1 { + s = queryTypes[0] + s.Version = draft04 + s.Description = descr + for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { + _, found := s.Properties.Get(pair.Key) if found { continue } - qt.Properties.Set(k, v) + s.Properties.Set(pair.Key, pair.Value) } + } else { + for _, qt := range queryTypes { + qt.Required = append(qt.Required, "refId") + + for k, v := range common { + _, found := qt.Properties.Get(k) + if found { + continue + } + qt.Properties.Set(k, v) + } - s.OneOf = append(s.OneOf, qt) + s.OneOf = append(s.OneOf, qt) + } } + if isRequest { + s = addRequestWrapper(s) + } body, err := json.MarshalIndent(s, "", " ") if err == nil { + // The invopop library should write to the kube-openapi flavor v := strings.Replace(string(body), `"$defs": {`, `"definitions": {`, 1) return []byte(v), nil } return body, err } +func addRequestWrapper(s *jsonschema.Schema) *jsonschema.Schema { + s.Definitions["query"] = &jsonschema.Schema{ + Properties: s.Properties, + AnyOf: s.AnyOf, + AllOf: s.AllOf, + OneOf: s.OneOf, + Type: "object", + } + s.AllOf = nil + s.AnyOf = nil + s.OneOf = nil + s.Properties = jsonschema.NewProperties() + s.Properties.Set("from", &jsonschema.Schema{ + Type: "string", + Description: "From Start time in epoch timestamps in milliseconds or relative using Grafana time units.", + }) + s.Properties.Set("to", &jsonschema.Schema{ + Type: "string", + Description: "To end time in epoch timestamps in milliseconds or relative using Grafana time units.", + }) + s.Properties.Set("queries", &jsonschema.Schema{ + Type: "array", + Items: &jsonschema.Schema{ + Ref: "#/definitions/query", + }, + }) + s.Properties.Set("debug", &jsonschema.Schema{ + Type: "boolean", + }) + s.Properties.Set("$schema", &jsonschema.Schema{ + Type: "string", + Description: "Optional schema URL -- this is not really used in production, but helpful for vscode debugging", + }) + s.AdditionalProperties = jsonschema.FalseSchema + s.Required = []string{"queries"} + s.Type = "object" + return s +} + func asJSONSchema(v any) (*jsonschema.Schema, error) { s, ok := v.(*jsonschema.Schema) if ok { diff --git a/experimental/spec/example/query.post.schema.json b/experimental/spec/example/query.post.schema.json deleted file mode 100644 index d4a204b56..000000000 --- a/experimental/spec/example/query.post.schema.json +++ /dev/null @@ -1,281 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-04/schema", - "definitions": { - "datasource": { - "properties": { - "type": { - "type": "string", - "description": "The datasource plugin type" - }, - "uid": { - "type": "string", - "description": "Datasource UID" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type", - "uid" - ], - "description": "The datasource" - }, - "hide": { - "type": "boolean", - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" - }, - "intervalMs": { - "type": "number", - "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" - }, - "maxDataPoints": { - "type": "integer", - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" - }, - "queryType": { - "type": "string", - "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." - }, - "refId": { - "type": "string", - "description": "RefID is the unique identifier of the query, set by the frontend call." - }, - "resultAssertions": { - "properties": { - "type": { - "type": "string", - "description": "Type asserts that the frame matches a known type structure." - }, - "typeVersion": { - "items": { - "type": "integer" - }, - "type": "array", - "maxItems": 2, - "minItems": 2, - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." - }, - "maxBytes": { - "type": "integer", - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" - }, - "maxFrames": { - "type": "integer", - "description": "Maximum frame count" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "typeVersion" - ], - "description": "Optionally define expected query result behavior" - }, - "timeRange": { - "properties": { - "from": { - "type": "string", - "description": "From is the start time of the query." - }, - "to": { - "type": "string", - "description": "To is the end time of the query." - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "from", - "to" - ], - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" - } - }, - "oneOf": [ - { - "properties": { - "expression": { - "type": "string", - "minLength": 1, - "description": "General math expression", - "examples": [ - "$A + 1", - "$A/$B" - ] - }, - "queryType": { - "type": "string", - "pattern": "^math$" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "refId": { - "$ref": "#/definitions/refId" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "queryType", - "refId" - ] - }, - { - "properties": { - "expression": { - "type": "string", - "description": "Reference to other query results" - }, - "reducer": { - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " - }, - "settings": { - "properties": { - "mode": { - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" - }, - "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "mode" - ], - "description": "Reducer Options" - }, - "queryType": { - "type": "string", - "pattern": "^reduce$" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "reducer", - "settings", - "queryType", - "refId" - ] - }, - { - "properties": { - "downsampler": { - "type": "string", - "description": "The reducer" - }, - "expression": { - "type": "string", - "description": "The math expression" - }, - "loadedDimensions": { - "additionalProperties": true, - "type": "object" - }, - "upsampler": { - "type": "string", - "description": "The reducer" - }, - "window": { - "type": "string", - "description": "A time duration string" - }, - "queryType": { - "type": "string", - "pattern": "^resample$" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions", - "queryType", - "refId" - ], - "description": "QueryType = resample" - } - ], - "properties": {}, - "type": "object", - "description": "Query model (the payload sent to /ds/query)" -} \ No newline at end of file diff --git a/experimental/spec/example/query.request.schema.json b/experimental/spec/example/query.request.schema.json new file mode 100644 index 000000000..234961978 --- /dev/null +++ b/experimental/spec/example/query.request.schema.json @@ -0,0 +1,311 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema", + "definitions": { + "datasource": { + "properties": { + "type": { + "type": "string", + "description": "The datasource plugin type" + }, + "uid": { + "type": "string", + "description": "Datasource UID" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type", + "uid" + ], + "description": "The datasource" + }, + "hide": { + "type": "boolean", + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" + }, + "intervalMs": { + "type": "number", + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" + }, + "maxDataPoints": { + "type": "integer", + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" + }, + "query": { + "oneOf": [ + { + "properties": { + "expression": { + "type": "string", + "minLength": 1, + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ] + }, + { + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " + }, + "settings": { + "properties": { + "mode": { + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" + }, + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "mode" + ], + "description": "Reducer Options" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ] + }, + { + "properties": { + "downsampler": { + "type": "string", + "description": "The reducer" + }, + "expression": { + "type": "string", + "description": "The math expression" + }, + "loadedDimensions": { + "additionalProperties": true, + "type": "object" + }, + "upsampler": { + "type": "string", + "description": "The reducer" + }, + "window": { + "type": "string", + "description": "A time duration string" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "$ref": "#/definitions/refId" + }, + "resultAssertions": { + "$ref": "#/definitions/resultAssertions" + }, + "timeRange": { + "$ref": "#/definitions/timeRange" + }, + "datasource": { + "$ref": "#/definitions/datasource" + }, + "maxDataPoints": { + "$ref": "#/definitions/maxDataPoints" + }, + "intervalMs": { + "$ref": "#/definitions/intervalMs" + }, + "hide": { + "$ref": "#/definitions/hide" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "description": "QueryType = resample" + } + ], + "properties": {}, + "type": "object" + }, + "queryType": { + "type": "string", + "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." + }, + "refId": { + "type": "string", + "description": "RefID is the unique identifier of the query, set by the frontend call." + }, + "resultAssertions": { + "properties": { + "type": { + "type": "string", + "description": "Type asserts that the frame matches a known type structure." + }, + "typeVersion": { + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2, + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." + }, + "maxBytes": { + "type": "integer", + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" + }, + "maxFrames": { + "type": "integer", + "description": "Maximum frame count" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "typeVersion" + ], + "description": "Optionally define expected query result behavior" + }, + "timeRange": { + "properties": { + "from": { + "type": "string", + "description": "From is the start time of the query." + }, + "to": { + "type": "string", + "description": "To is the end time of the query." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "from", + "to" + ], + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" + } + }, + "properties": { + "from": { + "type": "string", + "description": "From Start time in epoch timestamps in milliseconds or relative using Grafana time units." + }, + "to": { + "type": "string", + "description": "To end time in epoch timestamps in milliseconds or relative using Grafana time units." + }, + "queries": { + "items": { + "$ref": "#/definitions/query" + }, + "type": "array" + }, + "debug": { + "type": "boolean" + }, + "$schema": { + "type": "string", + "description": "Optional schema URL -- this is not really used in production, but helpful for vscode debugging" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "queries" + ], + "description": "Datasource request model" +} \ No newline at end of file diff --git a/experimental/spec/example/query.save.schema.json b/experimental/spec/example/query.schema.json similarity index 97% rename from experimental/spec/example/query.save.schema.json rename to experimental/spec/example/query.schema.json index 56d018526..4d6ac1cb5 100644 --- a/experimental/spec/example/query.save.schema.json +++ b/experimental/spec/example/query.schema.json @@ -219,6 +219,9 @@ "type": "string", "pattern": "^resample$" }, + "hide": { + "$ref": "#/definitions/hide" + }, "refId": { "$ref": "#/definitions/refId" }, @@ -230,9 +233,6 @@ }, "datasource": { "$ref": "#/definitions/datasource" - }, - "hide": { - "$ref": "#/definitions/hide" } }, "additionalProperties": false, @@ -249,11 +249,7 @@ "description": "QueryType = resample" } ], - "properties": { - "$schema": { - "type": "string" - } - }, + "properties": {}, "type": "object", - "description": "Save model (the payload saved in dashboards and alerts)" + "description": "object saved in dashboard/alert" } \ No newline at end of file diff --git a/experimental/spec/example/testdata/sample_query_request.json b/experimental/spec/example/testdata/sample_query_request.json index 1ef064a3c..e64f064d8 100644 --- a/experimental/spec/example/testdata/sample_query_request.json +++ b/experimental/spec/example/testdata/sample_query_request.json @@ -1,8 +1,31 @@ { - "$schema": "https://raw.githubusercontent.com/grafana/grafana-plugin-sdk-go/c67324318b0b055e73b038cb5eaf3090dc8b953c/experimental/spec/example/query.post.schema.json", - "refId": "A", - "queryType": "math", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A + 10" -} \ No newline at end of file + "from": "now-1h", + "to": "now", + "queries": [ + { + "refId": "A", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A + 10" + }, + { + "refId": "B", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A - $B" + }, + { + "refId": "C", + "queryType": "reduce", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + ] +} From 64c06c2cc1abbd70be63a2d1c92b46adaaad09e0 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sat, 24 Feb 2024 22:35:14 -0800 Subject: [PATCH 23/71] cleanup --- experimental/spec/builder.go | 4 ++++ experimental/spec/query_parser.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go index a2e54e27e..b383cda2d 100644 --- a/experimental/spec/builder.go +++ b/experimental/spec/builder.go @@ -465,6 +465,10 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, isR } func addRequestWrapper(s *jsonschema.Schema) *jsonschema.Schema { + if s.Definitions == nil { + s.Definitions = jsonschema.Definitions{} + } + s.Definitions["query"] = &jsonschema.Schema{ Properties: s.Properties, AnyOf: s.AnyOf, diff --git a/experimental/spec/query_parser.go b/experimental/spec/query_parser.go index abfe86bb7..26461da98 100644 --- a/experimental/spec/query_parser.go +++ b/experimental/spec/query_parser.go @@ -175,7 +175,7 @@ func (g *GenericDataQuery) Get(key string) (any, bool) { type genericQueryCodec struct{} -func (codec *genericQueryCodec) IsEmpty(ptr unsafe.Pointer) bool { +func (codec *genericQueryCodec) IsEmpty(_ unsafe.Pointer) bool { return false } From 36b841cd7409b4dfb11b96acfe5dbe186dd3d0b2 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 07:52:50 -0800 Subject: [PATCH 24/71] use kube-openapi --- experimental/spec/builder.go | 166 ++--- experimental/spec/example/query.go | 9 - ...quest.json => query.request.examples.json} | 2 +- .../spec/example/query.request.schema.json | 697 ++++++++++-------- experimental/spec/example/query.schema.json | 436 +++++++---- experimental/spec/example/query.types.json | 18 +- experimental/spec/example/query_test.go | 12 +- go.mod | 7 +- go.sum | 14 + 9 files changed, 783 insertions(+), 578 deletions(-) rename experimental/spec/example/{testdata/sample_query_request.json => query.request.examples.json} (99%) diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go index b383cda2d..663107f44 100644 --- a/experimental/spec/builder.go +++ b/experimental/spec/builder.go @@ -15,6 +15,9 @@ import ( "github.com/invopop/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/kube-openapi/pkg/validation/spec" + "k8s.io/kube-openapi/pkg/validation/strfmt" + "k8s.io/kube-openapi/pkg/validation/validate" ) // The k8s compatible jsonschema version @@ -95,7 +98,7 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { s := &jsonschema.Schema{ Type: "string", Extras: map[string]any{ - "x-enum-dictionary": enumValueDescriptions, + "x-enum-description": enumValueDescriptions, }, } for _, val := range f.Values { @@ -195,7 +198,7 @@ var whitespaceRegex = regexp.MustCompile(`\s+`) func (b *Builder) enumify(s *jsonschema.Schema) { if len(s.Enum) > 0 && s.Extras != nil { - extra, ok := s.Extras["x-enum-dictionary"] + extra, ok := s.Extras["x-enum-description"] if !ok { return } @@ -282,34 +285,44 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) QueryTypeDe } maybeUpdateFile(t, outfile, defs, body) - // Make sure the sample queries are actually valid - _, err = GetExampleQueries(defs) + // Read common query properties + query := &spec.Schema{} + err = query.UnmarshalJSON(GetCommonJSONSchema()) require.NoError(t, err) - // Read query info - query := &jsonschema.Schema{} - err = query.UnmarshalJSON(GetCommonJSONSchema()) + // Update the query save model schema + //------------------------------------ + outfile = filepath.Join(outdir, "query.schema.json") + schema, err := toQuerySchema(query, defs, false) require.NoError(t, err) - query.Version = draft04 // used by kube-openapi - query.Description = "Query properties shared by all data sources" - // Now update the query files - //---------------------------- + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, schema, body) + + // Update the request payload schema + //------------------------------------ outfile = filepath.Join(outdir, "query.request.schema.json") - schema, err := toQuerySchema(query, defs, true) + schema, err = toQuerySchema(query, defs, true) require.NoError(t, err) body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, schema, body) - // Now update the query files - //---------------------------- - outfile = filepath.Join(outdir, "query.schema.json") - schema, err = toQuerySchema(query, defs, false) + // Verify that the example queries actually validate + //------------------------------------ + request, err := GetExampleQueries(defs) require.NoError(t, err) + validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) + result := validator.Validate(request) + require.False(t, result.HasErrorsOrWarnings()) + require.True(t, result.MatchCount > 0, "must have some rules") + fmt.Printf("Validation: %+v\n", result) + + outfile = filepath.Join(outdir, "query.request.examples.json") body, _ = os.ReadFile(outfile) - maybeUpdateFile(t, outfile, schema, body) + maybeUpdateFile(t, outfile, request, body) + return defs } @@ -373,28 +386,25 @@ func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) Setting // Converts a set of queries into a single real schema (merged with the common properties) // This returns a the raw bytes because `invopop/jsonschema` requires extra manipulation // so that the results are readable by `kubernetes/kube-openapi` -func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, isRequest bool) (json.RawMessage, error) { +func toQuerySchema(generic *spec.Schema, defs QueryTypeDefinitionList, isRequest bool) (*spec.Schema, error) { descr := "object saved in dashboard/alert" if isRequest { descr = "Datasource request model" } ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true} - definitions := make(jsonschema.Definitions) - common := make(map[string]*jsonschema.Schema) - for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { - if !isRequest && ignoreForSave[pair.Key] { + common := make(map[string]spec.Schema) + for key, val := range generic.Properties { + if !isRequest && ignoreForSave[key] { continue // } - definitions[pair.Key] = pair.Value - common[pair.Key] = &jsonschema.Schema{Ref: "#/definitions/" + pair.Key} + common[key] = val } // The types for each query type - queryTypes := []*jsonschema.Schema{} + queryTypes := []*spec.Schema{} for _, qt := range defs.Items { node, err := asJSONSchema(qt.Spec.QuerySchema) - node.Version = "" if err != nil { return nil, fmt.Errorf("error reading query types schema: %s // %w", qt.ObjectMeta.Name, err) } @@ -404,111 +414,83 @@ func toQuerySchema(generic *jsonschema.Schema, defs QueryTypeDefinitionList, isR // Match all discriminators for _, d := range qt.Spec.Discriminators { - ds, ok := node.Properties.Get(d.Field) + ds, ok := node.Properties[d.Field] if !ok { - ds = &jsonschema.Schema{Type: "string"} - node.Properties.Set(d.Field, ds) + ds = *spec.StringProperty() } ds.Pattern = `^` + d.Value + `$` + node.Properties[d.Field] = ds node.Required = append(node.Required, d.Field) } queryTypes = append(queryTypes, node) } - s := &jsonschema.Schema{ - Type: "object", - Version: draft04, - Properties: jsonschema.NewProperties(), - Definitions: definitions, - Description: descr, + s := &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Schema: draft04, + Properties: make(map[string]spec.Schema), + Description: descr, + }, } // Single node -- just union the global and local properties if len(queryTypes) == 1 { s = queryTypes[0] - s.Version = draft04 + s.Schema = draft04 s.Description = descr - for pair := generic.Properties.Oldest(); pair != nil; pair = pair.Next() { - _, found := s.Properties.Get(pair.Key) + for key, val := range generic.Properties { + _, found := s.Properties[key] if found { continue } - s.Properties.Set(pair.Key, pair.Value) + s.Properties[key] = val } } else { for _, qt := range queryTypes { qt.Required = append(qt.Required, "refId") for k, v := range common { - _, found := qt.Properties.Get(k) + _, found := qt.Properties[k] if found { continue } - qt.Properties.Set(k, v) + qt.Properties[k] = v } - s.OneOf = append(s.OneOf, qt) + s.OneOf = append(s.OneOf, *qt) } } if isRequest { s = addRequestWrapper(s) } - body, err := json.MarshalIndent(s, "", " ") - if err == nil { - // The invopop library should write to the kube-openapi flavor - v := strings.Replace(string(body), `"$defs": {`, `"definitions": {`, 1) - return []byte(v), nil - } - return body, err + return s, nil } -func addRequestWrapper(s *jsonschema.Schema) *jsonschema.Schema { - if s.Definitions == nil { - s.Definitions = jsonschema.Definitions{} - } - - s.Definitions["query"] = &jsonschema.Schema{ - Properties: s.Properties, - AnyOf: s.AnyOf, - AllOf: s.AllOf, - OneOf: s.OneOf, - Type: "object", - } - s.AllOf = nil - s.AnyOf = nil - s.OneOf = nil - s.Properties = jsonschema.NewProperties() - s.Properties.Set("from", &jsonschema.Schema{ - Type: "string", - Description: "From Start time in epoch timestamps in milliseconds or relative using Grafana time units.", - }) - s.Properties.Set("to", &jsonschema.Schema{ - Type: "string", - Description: "To end time in epoch timestamps in milliseconds or relative using Grafana time units.", - }) - s.Properties.Set("queries", &jsonschema.Schema{ - Type: "array", - Items: &jsonschema.Schema{ - Ref: "#/definitions/query", +// moves the schema the the query slot in a request +func addRequestWrapper(s *spec.Schema) *spec.Schema { + return &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Required: []string{"queries"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: false}, + Properties: map[string]spec.Schema{ + "from": *spec.StringProperty().WithDescription( + "From Start time in epoch timestamps in milliseconds or relative using Grafana time units."), + "to": *spec.StringProperty().WithDescription( + "To end time in epoch timestamps in milliseconds or relative using Grafana time units."), + "queries": *spec.ArrayProperty(s), + "debug": *spec.BoolProperty(), + "$schema": *spec.StringProperty().WithDescription("helper"), + }, }, - }) - s.Properties.Set("debug", &jsonschema.Schema{ - Type: "boolean", - }) - s.Properties.Set("$schema", &jsonschema.Schema{ - Type: "string", - Description: "Optional schema URL -- this is not really used in production, but helpful for vscode debugging", - }) - s.AdditionalProperties = jsonschema.FalseSchema - s.Required = []string{"queries"} - s.Type = "object" - return s + } } -func asJSONSchema(v any) (*jsonschema.Schema, error) { - s, ok := v.(*jsonschema.Schema) +func asJSONSchema(v any) (*spec.Schema, error) { + s, ok := v.(*spec.Schema) if ok { return s, nil } @@ -516,7 +498,7 @@ func asJSONSchema(v any) (*jsonschema.Schema, error) { if err != nil { return nil, err } - s = &jsonschema.Schema{} + s = &spec.Schema{} err = json.Unmarshal(b, s) return s, err } diff --git a/experimental/spec/example/query.go b/experimental/spec/example/query.go index fba0c0c1d..47e3b1037 100644 --- a/experimental/spec/example/query.go +++ b/experimental/spec/example/query.go @@ -1,8 +1,6 @@ package example import ( - "embed" - "encoding/json" "fmt" "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" @@ -33,13 +31,6 @@ var _ spec.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) type QueyHandler struct{} -//go:embed query.types.json -var f embed.FS - -func (*QueyHandler) QueryTypeDefinitionsJSON() (json.RawMessage, error) { - return f.ReadFile("query.types.json") -} - // ReadQuery implements query.TypedQueryHandler. func (*QueyHandler) ParseQuery( // Properties that have been parsed off the same node diff --git a/experimental/spec/example/testdata/sample_query_request.json b/experimental/spec/example/query.request.examples.json similarity index 99% rename from experimental/spec/example/testdata/sample_query_request.json rename to experimental/spec/example/query.request.examples.json index e64f064d8..871d54dc8 100644 --- a/experimental/spec/example/testdata/sample_query_request.json +++ b/experimental/spec/example/query.request.examples.json @@ -28,4 +28,4 @@ } } ] -} +} \ No newline at end of file diff --git a/experimental/spec/example/query.request.schema.json b/experimental/spec/example/query.request.schema.json index 234961978..a9a381720 100644 --- a/experimental/spec/example/query.request.schema.json +++ b/experimental/spec/example/query.request.schema.json @@ -1,311 +1,420 @@ { - "$schema": "https://json-schema.org/draft-04/schema", - "definitions": { - "datasource": { - "properties": { - "type": { - "type": "string", - "description": "The datasource plugin type" - }, - "uid": { - "type": "string", - "description": "Datasource UID" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type", - "uid" - ], - "description": "The datasource" - }, - "hide": { - "type": "boolean", - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" + "type": "object", + "required": [ + "queries" + ], + "properties": { + "$schema": { + "description": "helper", + "type": "string" }, - "intervalMs": { - "type": "number", - "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" + "debug": { + "type": "boolean" }, - "maxDataPoints": { - "type": "integer", - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" + "from": { + "description": "From Start time in epoch timestamps in milliseconds or relative using Grafana time units.", + "type": "string" }, - "query": { - "oneOf": [ - { - "properties": { - "expression": { - "type": "string", - "minLength": 1, - "description": "General math expression", - "examples": [ - "$A + 1", - "$A/$B" - ] - }, - "queryType": { - "type": "string", - "pattern": "^math$" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" + "queries": { + "type": "array", + "items": { + "description": "Datasource request model", + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "expression": { + "description": "General math expression", + "type": "string", + "minLength": 1, + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "intervalMs": { + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", + "type": "number" + }, + "maxDataPoints": { + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", + "type": "integer" + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string" + }, + "to": { + "description": "To is the end time of the query.", + "type": "string" + } + }, + "additionalProperties": false + } }, - "datasource": { - "$ref": "#/definitions/datasource" - } + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "queryType", - "refId" - ] - }, - { - "properties": { - "expression": { - "type": "string", - "description": "Reference to other query results" - }, - "reducer": { - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " - }, - "settings": { - "properties": { - "mode": { - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" + { + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } }, - "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" + "additionalProperties": false + }, + "expression": { + "description": "Reference to other query results", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "intervalMs": { + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", + "type": "number" + }, + "maxDataPoints": { + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", + "type": "integer" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "reducer": { + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" } }, - "additionalProperties": false, - "type": "object", - "required": [ - "mode" - ], - "description": "Reducer Options" - }, - "queryType": { - "type": "string", - "pattern": "^reduce$" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "settings": { + "description": "Reducer Options", + "type": "object", + "required": [ + "mode" + ], + "properties": { + "mode": { + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "x-enum-description": { + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" + } + }, + "replaceWithValue": { + "description": "Only valid when mode is replace", + "type": "number" + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string" + }, + "to": { + "description": "To is the end time of the query.", + "type": "string" + } + }, + "additionalProperties": false + } }, - "hide": { - "$ref": "#/definitions/hide" - } + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "reducer", - "settings", - "queryType", - "refId" - ] - }, - { - "properties": { - "downsampler": { - "type": "string", - "description": "The reducer" - }, - "expression": { - "type": "string", - "description": "The math expression" - }, - "loadedDimensions": { - "additionalProperties": true, - "type": "object" - }, - "upsampler": { - "type": "string", - "description": "The reducer" - }, - "window": { - "type": "string", - "description": "A time duration string" - }, - "queryType": { - "type": "string", - "pattern": "^resample$" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - }, - "maxDataPoints": { - "$ref": "#/definitions/maxDataPoints" - }, - "intervalMs": { - "$ref": "#/definitions/intervalMs" + { + "description": "QueryType = resample", + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "downsampler": { + "description": "The reducer", + "type": "string" + }, + "expression": { + "description": "The math expression", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "intervalMs": { + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", + "type": "number" + }, + "loadedDimensions": { + "type": "object", + "additionalProperties": true, + "x-grafana-type": "data.DataFrame" + }, + "maxDataPoints": { + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", + "type": "integer" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string" + }, + "to": { + "description": "To is the end time of the query.", + "type": "string" + } + }, + "additionalProperties": false + }, + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "A time duration string", + "type": "string" + } }, - "hide": { - "$ref": "#/definitions/hide" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions", - "queryType", - "refId" - ], - "description": "QueryType = resample" - } - ], - "properties": {}, - "type": "object" - }, - "queryType": { - "type": "string", - "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." - }, - "refId": { - "type": "string", - "description": "RefID is the unique identifier of the query, set by the frontend call." - }, - "resultAssertions": { - "properties": { - "type": { - "type": "string", - "description": "Type asserts that the frame matches a known type structure." - }, - "typeVersion": { - "items": { - "type": "integer" - }, - "type": "array", - "maxItems": 2, - "minItems": 2, - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." - }, - "maxBytes": { - "type": "integer", - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" - }, - "maxFrames": { - "type": "integer", - "description": "Maximum frame count" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "typeVersion" - ], - "description": "Optionally define expected query result behavior" - }, - "timeRange": { - "properties": { - "from": { - "type": "string", - "description": "From is the start time of the query." - }, - "to": { - "type": "string", - "description": "To is the end time of the query." - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "from", - "to" - ], - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" - } - }, - "properties": { - "from": { - "type": "string", - "description": "From Start time in epoch timestamps in milliseconds or relative using Grafana time units." + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + } + ], + "$schema": "https://json-schema.org/draft-04/schema" + } }, "to": { - "type": "string", - "description": "To end time in epoch timestamps in milliseconds or relative using Grafana time units." - }, - "queries": { - "items": { - "$ref": "#/definitions/query" - }, - "type": "array" - }, - "debug": { - "type": "boolean" - }, - "$schema": { - "type": "string", - "description": "Optional schema URL -- this is not really used in production, but helpful for vscode debugging" + "description": "To end time in epoch timestamps in milliseconds or relative using Grafana time units.", + "type": "string" } }, - "additionalProperties": false, - "type": "object", - "required": [ - "queries" - ], - "description": "Datasource request model" + "additionalProperties": false } \ No newline at end of file diff --git a/experimental/spec/example/query.schema.json b/experimental/spec/example/query.schema.json index 4d6ac1cb5..dcb5bff32 100644 --- a/experimental/spec/example/query.schema.json +++ b/experimental/spec/example/query.schema.json @@ -1,135 +1,152 @@ { - "$schema": "https://json-schema.org/draft-04/schema", - "definitions": { - "datasource": { - "properties": { - "type": { - "type": "string", - "description": "The datasource plugin type" - }, - "uid": { - "type": "string", - "description": "Datasource UID" - } - }, - "additionalProperties": false, + "description": "object saved in dashboard/alert", + "type": "object", + "oneOf": [ + { "type": "object", "required": [ - "type", - "uid" + "expression", + "queryType", + "refId" ], - "description": "The datasource" - }, - "hide": { - "type": "boolean", - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" - }, - "queryType": { - "type": "string", - "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." - }, - "refId": { - "type": "string", - "description": "RefID is the unique identifier of the query, set by the frontend call." - }, - "resultAssertions": { "properties": { - "type": { - "type": "string", - "description": "Type asserts that the frame matches a known type structure." - }, - "typeVersion": { - "items": { - "type": "integer" + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } }, - "type": "array", - "maxItems": 2, - "minItems": 2, - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." - }, - "maxBytes": { - "type": "integer", - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" - }, - "maxFrames": { - "type": "integer", - "description": "Maximum frame count" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "typeVersion" - ], - "description": "Optionally define expected query result behavior" - }, - "timeRange": { - "properties": { - "from": { - "type": "string", - "description": "From is the start time of the query." + "additionalProperties": false }, - "to": { - "type": "string", - "description": "To is the end time of the query." - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "from", - "to" - ], - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" - } - }, - "oneOf": [ - { - "properties": { "expression": { + "description": "General math expression", "type": "string", "minLength": 1, - "description": "General math expression", "examples": [ "$A + 1", "$A/$B" ] }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, "queryType": { "type": "string", "pattern": "^math$" }, "refId": { - "$ref": "#/definitions/refId" + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" }, "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false }, "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" - }, - "hide": { - "$ref": "#/definitions/hide" + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string" + }, + "to": { + "description": "To is the end time of the query.", + "type": "string" + } + }, + "additionalProperties": false } }, "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + }, + { "type": "object", "required": [ "expression", + "reducer", + "settings", "queryType", "refId" - ] - }, - { + ], "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, "expression": { + "description": "Reference to other query results", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "queryType": { "type": "string", - "description": "Reference to other query results" + "pattern": "^reduce$" }, "reducer": { + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", "type": "string", "enum": [ "sum", @@ -139,117 +156,214 @@ "count", "last" ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` " + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" + } + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false }, "settings": { + "description": "Reducer Options", + "type": "object", + "required": [ + "mode" + ], "properties": { "mode": { + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", "type": "string", "enum": [ "dropNN", "replaceNN" ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers" + "x-enum-description": { + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" + } }, "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" + "description": "Only valid when mode is replace", + "type": "number" } }, - "additionalProperties": false, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", "type": "object", "required": [ - "mode" + "from", + "to" ], - "description": "Reducer Options" - }, - "queryType": { - "type": "string", - "pattern": "^reduce$" - }, - "hide": { - "$ref": "#/definitions/hide" - }, - "refId": { - "$ref": "#/definitions/refId" - }, - "resultAssertions": { - "$ref": "#/definitions/resultAssertions" - }, - "timeRange": { - "$ref": "#/definitions/timeRange" - }, - "datasource": { - "$ref": "#/definitions/datasource" + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string" + }, + "to": { + "description": "To is the end time of the query.", + "type": "string" + } + }, + "additionalProperties": false } }, "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + }, + { + "description": "QueryType = resample", "type": "object", "required": [ "expression", - "reducer", - "settings", + "window", + "downsampler", + "upsampler", + "loadedDimensions", "queryType", "refId" - ] - }, - { + ], "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, "downsampler": { - "type": "string", - "description": "The reducer" + "description": "The reducer", + "type": "string" }, "expression": { - "type": "string", - "description": "The math expression" + "description": "The math expression", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" }, "loadedDimensions": { + "type": "object", "additionalProperties": true, - "type": "object" - }, - "upsampler": { - "type": "string", - "description": "The reducer" - }, - "window": { - "type": "string", - "description": "A time duration string" + "x-grafana-type": "data.DataFrame" }, "queryType": { "type": "string", "pattern": "^resample$" }, - "hide": { - "$ref": "#/definitions/hide" - }, "refId": { - "$ref": "#/definitions/refId" + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" }, "resultAssertions": { - "$ref": "#/definitions/resultAssertions" + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false }, "timeRange": { - "$ref": "#/definitions/timeRange" + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string" + }, + "to": { + "description": "To is the end time of the query.", + "type": "string" + } + }, + "additionalProperties": false }, - "datasource": { - "$ref": "#/definitions/datasource" + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "A time duration string", + "type": "string" } }, "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions", - "queryType", - "refId" - ], - "description": "QueryType = resample" + "$schema": "https://json-schema.org/draft-04/schema" } ], - "properties": {}, - "type": "object", - "description": "object saved in dashboard/alert" + "$schema": "https://json-schema.org/draft-04/schema" } \ No newline at end of file diff --git a/experimental/spec/example/query.types.json b/experimental/spec/example/query.types.json index 5a5982001..a8d296764 100644 --- a/experimental/spec/example/query.types.json +++ b/experimental/spec/example/query.types.json @@ -20,22 +20,22 @@ ], "querySchema": { "$schema": "https://json-schema.org/draft-04/schema", + "additionalProperties": false, "properties": { "expression": { - "type": "string", - "minLength": 1, "description": "General math expression", "examples": [ "$A + 1", "$A/$B" - ] + ], + "minLength": 1, + "type": "string" } }, - "additionalProperties": false, - "type": "object", "required": [ "expression" - ] + ], + "type": "object" }, "examples": [ { @@ -56,7 +56,7 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1708817676338", + "resourceVersion": "1708876267448", "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { @@ -84,7 +84,7 @@ "last" ], "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", - "x-enum-dictionary": { + "x-enum-description": { "mean": "The mean", "sum": "The sum" } @@ -98,7 +98,7 @@ "replaceNN" ], "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "x-enum-dictionary": { + "x-enum-description": { "dropNN": "Drop non-numbers", "replaceNN": "Replace non-numbers" } diff --git a/experimental/spec/example/query_test.go b/experimental/spec/example/query_test.go index c18f391d5..e298852c5 100644 --- a/experimental/spec/example/query_test.go +++ b/experimental/spec/example/query_test.go @@ -1,8 +1,6 @@ package example import ( - "encoding/json" - "fmt" "reflect" "testing" @@ -62,13 +60,5 @@ func TestQueryTypeDefinitions(t *testing.T) { }) require.NoError(t, err) - defs := builder.UpdateQueryDefinition(t, "./") - - queries, err := spec.GetExampleQueries(defs) - require.NoError(t, err) - - out, err := json.MarshalIndent(queries, "", " ") - require.NoError(t, err) - - fmt.Printf("%s", string(out)) + _ = builder.UpdateQueryDefinition(t, "./") } diff --git a/go.mod b/go.mod index 09acd050b..4ab294250 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( google.golang.org/grpc v1.60.1 google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20240220201932-37d671a357a5 // @grafana/grafana-app-platform-squad ) require ( @@ -36,6 +37,7 @@ require ( github.com/getkin/kin-openapi v0.120.0 github.com/go-jose/go-jose/v3 v3.0.1 github.com/google/uuid v1.6.0 + github.com/invopop/jsonschema v0.12.0 github.com/unknwon/bra v0.0.0-20200517080246-1e3013ecaff8 github.com/urfave/cli v1.22.14 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 @@ -55,6 +57,7 @@ require ( require ( github.com/BurntSushi/toml v1.3.2 // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect @@ -67,13 +70,14 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.4 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 // indirect - github.com/invopop/jsonschema v0.12.0 // indirect github.com/invopop/yaml v0.2.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect @@ -108,4 +112,5 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect ) diff --git a/go.sum b/go.sum index 60ee2326c..48d28e623 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/apache/arrow/go/v15 v15.0.0 h1:1zZACWf85oEZY5/kd9dsQS7i+2G5zVQcbKTHgslqHNA= github.com/apache/arrow/go/v15 v15.0.0/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -70,6 +72,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= @@ -93,12 +97,16 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= @@ -401,3 +409,9 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/kube-openapi v0.0.0-20240220201932-37d671a357a5 h1:QSpdNrZ9uRlV0VkqLvVO0Rqg8ioKi3oSw7O5P7pJV8M= +k8s.io/kube-openapi v0.0.0-20240220201932-37d671a357a5/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= From 2c8c4ada2764cfeb8f42b7fc8221ec43cb8a743a Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 07:59:48 -0800 Subject: [PATCH 25/71] fix error state --- experimental/spec/builder.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go index 663107f44..a0ac874ba 100644 --- a/experimental/spec/builder.go +++ b/experimental/spec/builder.go @@ -313,16 +313,19 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) QueryTypeDe request, err := GetExampleQueries(defs) require.NoError(t, err) - validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) - result := validator.Validate(request) - require.False(t, result.HasErrorsOrWarnings()) - require.True(t, result.MatchCount > 0, "must have some rules") - fmt.Printf("Validation: %+v\n", result) - outfile = filepath.Join(outdir, "query.request.examples.json") body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, request, body) + validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) + result := validator.Validate(request) + if result.HasErrorsOrWarnings() { + body, err = json.MarshalIndent(result, "", " ") + require.NoError(t, err) + fmt.Printf("Validation: %s\n", string(body)) + require.Fail(t, "validation failed") + } + require.True(t, result.MatchCount > 0, "must have some rules") return defs } From 2b78392d8aa0b1ae8f93da7a3a9c19a097433fbd Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 14:14:00 -0800 Subject: [PATCH 26/71] refactor (again) --- data/frame_type.go | 1 + experimental/spec/builder.go | 575 ------------------ experimental/spec/enums.go | 112 ---- experimental/spec/enums_test.go | 22 - experimental/spec/example/math.go | 46 -- experimental/spec/example/query.go | 55 -- .../spec/example/query.request.examples.json | 31 - .../spec/example/query.request.schema.json | 420 ------------- experimental/spec/example/query.schema.json | 369 ----------- experimental/spec/example/query.types.json | 193 ------ experimental/spec/example/query_test.go | 64 -- experimental/spec/example/reduce.go | 57 -- experimental/spec/example/resample.go | 28 - experimental/spec/k8s.go | 52 -- experimental/spec/query.go | 152 ----- experimental/spec/query.schema.json | 98 --- experimental/spec/query_parser.go | 322 ---------- experimental/spec/query_test.go | 50 -- experimental/spec/settings.go | 23 - experimental/testdata/folder.golden.txt | 2 +- 20 files changed, 2 insertions(+), 2670 deletions(-) delete mode 100644 experimental/spec/builder.go delete mode 100644 experimental/spec/enums.go delete mode 100644 experimental/spec/enums_test.go delete mode 100644 experimental/spec/example/math.go delete mode 100644 experimental/spec/example/query.go delete mode 100644 experimental/spec/example/query.request.examples.json delete mode 100644 experimental/spec/example/query.request.schema.json delete mode 100644 experimental/spec/example/query.schema.json delete mode 100644 experimental/spec/example/query.types.json delete mode 100644 experimental/spec/example/query_test.go delete mode 100644 experimental/spec/example/reduce.go delete mode 100644 experimental/spec/example/resample.go delete mode 100644 experimental/spec/k8s.go delete mode 100644 experimental/spec/query.go delete mode 100644 experimental/spec/query.schema.json delete mode 100644 experimental/spec/query_parser.go delete mode 100644 experimental/spec/query_test.go delete mode 100644 experimental/spec/settings.go diff --git a/data/frame_type.go b/data/frame_type.go index fa2f41ded..1558b7327 100644 --- a/data/frame_type.go +++ b/data/frame_type.go @@ -10,6 +10,7 @@ import ( // frame's structure conforms to the FrameType's specification. // This property is currently optional, so FrameType may be FrameTypeUnknown even if the properties of // the Frame correspond to a defined FrameType. +// +enum type FrameType string // --- diff --git a/experimental/spec/builder.go b/experimental/spec/builder.go deleted file mode 100644 index a0ac874ba..000000000 --- a/experimental/spec/builder.go +++ /dev/null @@ -1,575 +0,0 @@ -package spec - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "reflect" - "regexp" - "strings" - "testing" - "time" - - "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/invopop/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "k8s.io/kube-openapi/pkg/validation/spec" - "k8s.io/kube-openapi/pkg/validation/strfmt" - "k8s.io/kube-openapi/pkg/validation/validate" -) - -// The k8s compatible jsonschema version -const draft04 = "https://json-schema.org/draft-04/schema" - -// SchemaBuilder is a helper function that can be used by -// backend build processes to produce static schema definitions -// This is not intended as runtime code, and is not the only way to -// produce a schema (we may also want/need to use typescript as the source) -type Builder struct { - opts BuilderOptions - reflector *jsonschema.Reflector // Needed to use comments - query []QueryTypeDefinition - setting []SettingsDefinition -} - -type BuilderOptions struct { - // ex "github.com/invopop/jsonschema" - BasePackage string - - // ex "./" - CodePath string - - // explicitly define the enumeration fields - Enums []reflect.Type -} - -type QueryTypeInfo struct { - // The management name - Name string - // Optional discriminators - Discriminators []DiscriminatorFieldValue - // Raw GO type used for reflection - GoType reflect.Type - // Add sample queries - Examples []QueryExample -} - -type SettingTypeInfo struct { - // The management name - Name string - // Optional discriminators - Discriminators []DiscriminatorFieldValue - // Raw GO type used for reflection - GoType reflect.Type - // Map[string]string - SecureGoType reflect.Type -} - -func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { - r := new(jsonschema.Reflector) - r.DoNotReference = true - if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { - return nil, err - } - customMapper := map[reflect.Type]*jsonschema.Schema{ - reflect.TypeOf(data.Frame{}): { - Type: "object", - Extras: map[string]any{ - "x-grafana-type": "data.DataFrame", - }, - AdditionalProperties: jsonschema.TrueSchema, - }, - } - r.Mapper = func(t reflect.Type) *jsonschema.Schema { - return customMapper[t] - } - - if len(opts.Enums) > 0 { - fields, err := findEnumFields(opts.BasePackage, opts.CodePath) - if err != nil { - return nil, err - } - for _, etype := range opts.Enums { - for _, f := range fields { - if f.Name == etype.Name() && f.Package == etype.PkgPath() { - enumValueDescriptions := map[string]string{} - s := &jsonschema.Schema{ - Type: "string", - Extras: map[string]any{ - "x-enum-description": enumValueDescriptions, - }, - } - for _, val := range f.Values { - s.Enum = append(s.Enum, val.Value) - if val.Comment != "" { - enumValueDescriptions[val.Value] = val.Comment - } - } - customMapper[etype] = s - } - } - } - } - - return &Builder{ - opts: opts, - reflector: r, - }, nil -} - -func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { - for _, info := range inputs { - schema := b.reflector.ReflectFromType(info.GoType) - if schema == nil { - return fmt.Errorf("missing schema") - } - - b.enumify(schema) - - name := info.Name - if name == "" { - for _, dis := range info.Discriminators { - if name != "" { - name += "-" - } - name += dis.Value - } - if name == "" { - return fmt.Errorf("missing name or discriminators") - } - } - - // We need to be careful to only use draft-04 so that this is possible to use - // with kube-openapi - schema.Version = draft04 - schema.ID = "" - schema.Anchor = "" - - b.query = append(b.query, QueryTypeDefinition{ - ObjectMeta: ObjectMeta{ - Name: name, - }, - Spec: QueryTypeDefinitionSpec{ - Discriminators: info.Discriminators, - QuerySchema: schema, - Examples: info.Examples, - }, - }) - } - return nil -} - -func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { - for _, info := range inputs { - name := info.Name - if name == "" { - return fmt.Errorf("missing name") - } - - schema := b.reflector.ReflectFromType(info.GoType) - if schema == nil { - return fmt.Errorf("missing schema") - } - - b.enumify(schema) - - // used by kube-openapi - schema.Version = draft04 - schema.ID = "" - schema.Anchor = "" - - b.setting = append(b.setting, SettingsDefinition{ - ObjectMeta: ObjectMeta{ - Name: name, - }, - Spec: SettingsDefinitionSpec{ - Discriminators: info.Discriminators, - JSONDataSchema: schema, - }, - }) - } - return nil -} - -// whitespaceRegex is the regex for consecutive whitespaces. -var whitespaceRegex = regexp.MustCompile(`\s+`) - -func (b *Builder) enumify(s *jsonschema.Schema) { - if len(s.Enum) > 0 && s.Extras != nil { - extra, ok := s.Extras["x-enum-description"] - if !ok { - return - } - - lookup, ok := extra.(map[string]string) - if !ok { - return - } - - lines := []string{} - if s.Description != "" { - lines = append(lines, s.Description, "\n") - } - lines = append(lines, "Possible enum values:") - for _, v := range s.Enum { - c := lookup[v.(string)] - c = whitespaceRegex.ReplaceAllString(c, " ") - lines = append(lines, fmt.Sprintf(" - `%q` %s", v, c)) - } - - s.Description = strings.Join(lines, "\n") - return - } - - for pair := s.Properties.Oldest(); pair != nil; pair = pair.Next() { - b.enumify(pair.Value) - } -} - -// Update the schema definition file -// When placed in `static/schema/query.types.json` folder of a plugin distribution, -// it can be used to advertise various query types -// If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) QueryTypeDefinitionList { - t.Helper() - - outfile := filepath.Join(outdir, "query.types.json") - now := time.Now().UTC() - rv := fmt.Sprintf("%d", now.UnixMilli()) - - defs := QueryTypeDefinitionList{} - byName := make(map[string]*QueryTypeDefinition) - body, err := os.ReadFile(outfile) - if err == nil { - err = json.Unmarshal(body, &defs) - if err == nil { - for i, def := range defs.Items { - byName[def.ObjectMeta.Name] = &defs.Items[i] - } - } - } - defs.Kind = "QueryTypeDefinitionList" - defs.APIVersion = "query.grafana.app/v0alpha1" - - // The updated schemas - for _, def := range b.query { - found, ok := byName[def.ObjectMeta.Name] - if !ok { - defs.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) - - defs.Items = append(defs.Items, def) - } else { - var o1, o2 interface{} - b1, _ := json.Marshal(def.Spec) - b2, _ := json.Marshal(found.Spec) - _ = json.Unmarshal(b1, &o1) - _ = json.Unmarshal(b2, &o2) - if !reflect.DeepEqual(o1, o2) { - found.ObjectMeta.ResourceVersion = rv - found.Spec = def.Spec - } - delete(byName, def.ObjectMeta.Name) - } - } - - if defs.ObjectMeta.ResourceVersion == "" { - defs.ObjectMeta.ResourceVersion = rv - } - - if len(byName) > 0 { - require.FailNow(t, "query type removed, manually update (for now)") - } - maybeUpdateFile(t, outfile, defs, body) - - // Read common query properties - query := &spec.Schema{} - err = query.UnmarshalJSON(GetCommonJSONSchema()) - require.NoError(t, err) - - // Update the query save model schema - //------------------------------------ - outfile = filepath.Join(outdir, "query.schema.json") - schema, err := toQuerySchema(query, defs, false) - require.NoError(t, err) - - body, _ = os.ReadFile(outfile) - maybeUpdateFile(t, outfile, schema, body) - - // Update the request payload schema - //------------------------------------ - outfile = filepath.Join(outdir, "query.request.schema.json") - schema, err = toQuerySchema(query, defs, true) - require.NoError(t, err) - - body, _ = os.ReadFile(outfile) - maybeUpdateFile(t, outfile, schema, body) - - // Verify that the example queries actually validate - //------------------------------------ - request, err := GetExampleQueries(defs) - require.NoError(t, err) - - outfile = filepath.Join(outdir, "query.request.examples.json") - body, _ = os.ReadFile(outfile) - maybeUpdateFile(t, outfile, request, body) - - validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) - result := validator.Validate(request) - if result.HasErrorsOrWarnings() { - body, err = json.MarshalIndent(result, "", " ") - require.NoError(t, err) - fmt.Printf("Validation: %s\n", string(body)) - require.Fail(t, "validation failed") - } - require.True(t, result.MatchCount > 0, "must have some rules") - return defs -} - -// Update the schema definition file -// When placed in `static/schema/query.schema.json` folder of a plugin distribution, -// it can be used to advertise various query types -// If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) SettingsDefinitionList { - t.Helper() - - now := time.Now().UTC() - rv := fmt.Sprintf("%d", now.UnixMilli()) - - defs := SettingsDefinitionList{} - byName := make(map[string]*SettingsDefinition) - body, err := os.ReadFile(outfile) - if err == nil { - err = json.Unmarshal(body, &defs) - if err == nil { - for i, def := range defs.Items { - byName[def.ObjectMeta.Name] = &defs.Items[i] - } - } - } - defs.Kind = "SettingsDefinitionList" - defs.APIVersion = "common.grafana.app/v0alpha1" - - // The updated schemas - for _, def := range b.setting { - found, ok := byName[def.ObjectMeta.Name] - if !ok { - defs.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) - - defs.Items = append(defs.Items, def) - } else { - var o1, o2 interface{} - b1, _ := json.Marshal(def.Spec) - b2, _ := json.Marshal(found.Spec) - _ = json.Unmarshal(b1, &o1) - _ = json.Unmarshal(b2, &o2) - if !reflect.DeepEqual(o1, o2) { - found.ObjectMeta.ResourceVersion = rv - found.Spec = def.Spec - } - delete(byName, def.ObjectMeta.Name) - } - } - - if defs.ObjectMeta.ResourceVersion == "" { - defs.ObjectMeta.ResourceVersion = rv - } - - if len(byName) > 0 { - require.FailNow(t, "settings type removed, manually update (for now)") - } - return defs -} - -// Converts a set of queries into a single real schema (merged with the common properties) -// This returns a the raw bytes because `invopop/jsonschema` requires extra manipulation -// so that the results are readable by `kubernetes/kube-openapi` -func toQuerySchema(generic *spec.Schema, defs QueryTypeDefinitionList, isRequest bool) (*spec.Schema, error) { - descr := "object saved in dashboard/alert" - if isRequest { - descr = "Datasource request model" - } - - ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true} - common := make(map[string]spec.Schema) - for key, val := range generic.Properties { - if !isRequest && ignoreForSave[key] { - continue // - } - common[key] = val - } - - // The types for each query type - queryTypes := []*spec.Schema{} - for _, qt := range defs.Items { - node, err := asJSONSchema(qt.Spec.QuerySchema) - if err != nil { - return nil, fmt.Errorf("error reading query types schema: %s // %w", qt.ObjectMeta.Name, err) - } - if node == nil { - return nil, fmt.Errorf("missing query schema: %s // %v", qt.ObjectMeta.Name, qt) - } - - // Match all discriminators - for _, d := range qt.Spec.Discriminators { - ds, ok := node.Properties[d.Field] - if !ok { - ds = *spec.StringProperty() - } - ds.Pattern = `^` + d.Value + `$` - node.Properties[d.Field] = ds - node.Required = append(node.Required, d.Field) - } - - queryTypes = append(queryTypes, node) - } - - s := &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Schema: draft04, - Properties: make(map[string]spec.Schema), - Description: descr, - }, - } - - // Single node -- just union the global and local properties - if len(queryTypes) == 1 { - s = queryTypes[0] - s.Schema = draft04 - s.Description = descr - for key, val := range generic.Properties { - _, found := s.Properties[key] - if found { - continue - } - s.Properties[key] = val - } - } else { - for _, qt := range queryTypes { - qt.Required = append(qt.Required, "refId") - - for k, v := range common { - _, found := qt.Properties[k] - if found { - continue - } - qt.Properties[k] = v - } - - s.OneOf = append(s.OneOf, *qt) - } - } - - if isRequest { - s = addRequestWrapper(s) - } - return s, nil -} - -// moves the schema the the query slot in a request -func addRequestWrapper(s *spec.Schema) *spec.Schema { - return &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Required: []string{"queries"}, - AdditionalProperties: &spec.SchemaOrBool{Allows: false}, - Properties: map[string]spec.Schema{ - "from": *spec.StringProperty().WithDescription( - "From Start time in epoch timestamps in milliseconds or relative using Grafana time units."), - "to": *spec.StringProperty().WithDescription( - "To end time in epoch timestamps in milliseconds or relative using Grafana time units."), - "queries": *spec.ArrayProperty(s), - "debug": *spec.BoolProperty(), - "$schema": *spec.StringProperty().WithDescription("helper"), - }, - }, - } -} - -func asJSONSchema(v any) (*spec.Schema, error) { - s, ok := v.(*spec.Schema) - if ok { - return s, nil - } - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - s = &spec.Schema{} - err = json.Unmarshal(b, s) - return s, err -} - -func asGenericDataQuery(v any) (*GenericDataQuery, error) { - s, ok := v.(*GenericDataQuery) - if ok { - return s, nil - } - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - s = &GenericDataQuery{} - err = json.Unmarshal(b, s) - return s, err -} - -func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { - t.Helper() - - out, err := json.MarshalIndent(value, "", " ") - require.NoError(t, err) - - update := false - if err == nil { - if !assert.JSONEq(t, string(out), string(body)) { - update = true - } - } else { - update = true - } - if update { - err = os.WriteFile(outfile, out, 0600) - require.NoError(t, err, "error writing file") - } -} - -func GetExampleQueries(defs QueryTypeDefinitionList) (QueryRequest[GenericDataQuery], error) { - rsp := QueryRequest[GenericDataQuery]{ - From: "now-1h", - To: "now", - Queries: []GenericDataQuery{}, - } - - for _, def := range defs.Items { - for _, sample := range def.Spec.Examples { - if sample.SaveModel != nil { - q, err := asGenericDataQuery(sample.SaveModel) - if err != nil { - return rsp, fmt.Errorf("invalid sample save query [%s], in %s // %w", - sample.Name, def.ObjectMeta.Name, err) - } - q.RefID = string(rune('A' + len(rsp.Queries))) - for _, dis := range def.Spec.Discriminators { - _ = q.Set(dis.Field, dis.Value) - } - - if q.MaxDataPoints < 1 { - q.MaxDataPoints = 1000 - } - if q.IntervalMS < 1 { - q.IntervalMS = 5 - } - - rsp.Queries = append(rsp.Queries, *q) - } - } - } - return rsp, nil -} diff --git a/experimental/spec/enums.go b/experimental/spec/enums.go deleted file mode 100644 index fb1ae0f2b..000000000 --- a/experimental/spec/enums.go +++ /dev/null @@ -1,112 +0,0 @@ -package spec - -import ( - "io/fs" - gopath "path" - "path/filepath" - "strings" - - "go/ast" - "go/doc" - "go/parser" - "go/token" -) - -type EnumValue struct { - Value string - Comment string -} - -type EnumField struct { - Package string - Name string - Comment string - Values []EnumValue -} - -func findEnumFields(base, path string) ([]EnumField, error) { - fset := token.NewFileSet() - dict := make(map[string][]*ast.Package) - err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - d, err := parser.ParseDir(fset, path, nil, parser.ParseComments) - if err != nil { - return err - } - for _, v := range d { - // paths may have multiple packages, like for tests - k := gopath.Join(base, path) - dict[k] = append(dict[k], v) - } - } - return nil - }) - if err != nil { - return nil, err - } - - fields := make([]EnumField, 0) - field := &EnumField{} - dp := &doc.Package{} - - for pkg, p := range dict { - for _, f := range p { - gtxt := "" - typ := "" - ast.Inspect(f, func(n ast.Node) bool { - switch x := n.(type) { - case *ast.TypeSpec: - typ = x.Name.String() - if !ast.IsExported(typ) { - typ = "" - } else { - txt := x.Doc.Text() - if txt == "" && gtxt != "" { - txt = gtxt - gtxt = "" - } - txt = strings.TrimSpace(dp.Synopsis(txt)) - if strings.HasSuffix(txt, "+enum") { - fields = append(fields, EnumField{ - Package: pkg, - Name: typ, - Comment: strings.TrimSpace(strings.TrimSuffix(txt, "+enum")), - }) - field = &fields[len(fields)-1] - } - } - case *ast.ValueSpec: - txt := x.Doc.Text() - if txt == "" { - txt = x.Comment.Text() - } - if typ == field.Name { - for _, n := range x.Names { - if ast.IsExported(n.String()) { - v, ok := x.Values[0].(*ast.BasicLit) - if ok { - val := strings.TrimPrefix(v.Value, `"`) - val = strings.TrimSuffix(val, `"`) - txt = strings.TrimSpace(txt) - field.Values = append(field.Values, EnumValue{ - Value: val, - Comment: txt, - }) - } - } - } - } - case *ast.GenDecl: - // remember for the next type - gtxt = x.Doc.Text() - } - return true - }) - } - } - - return fields, nil -} diff --git a/experimental/spec/enums_test.go b/experimental/spec/enums_test.go deleted file mode 100644 index dd9d27b5a..000000000 --- a/experimental/spec/enums_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package spec - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestFindEnums(t *testing.T) { - fields, err := findEnumFields( - "github.com/grafana/grafana-plugin-sdk-go/experimental/spec", - "./example") - require.NoError(t, err) - - out, err := json.MarshalIndent(fields, "", " ") - require.NoError(t, err) - fmt.Printf("%s", string(out)) - - require.Equal(t, 3, len(fields)) -} diff --git a/experimental/spec/example/math.go b/experimental/spec/example/math.go deleted file mode 100644 index 0ef5d43c6..000000000 --- a/experimental/spec/example/math.go +++ /dev/null @@ -1,46 +0,0 @@ -package example - -import "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - -var _ ExpressionQuery = (*MathQuery)(nil) - -type MathQuery struct { - // General math expression - Expression string `json:"expression" jsonschema:"minLength=1,example=$A + 1,example=$A/$B"` - - // Parsed from the expression - variables []string `json:"-"` -} - -func (*MathQuery) ExpressionQueryType() QueryType { - return QueryTypeMath -} - -func (q *MathQuery) Variables() []string { - return q.variables -} - -func readMathQuery(iter *jsoniter.Iterator) (*MathQuery, error) { - var q *MathQuery - var err error - fname := "" - for fname, err = iter.ReadObject(); fname != "" && err == nil; fname, err = iter.ReadObject() { - switch fname { - case "expression": - temp, err := iter.ReadString() - if err != nil { - return q, err - } - q = &MathQuery{ - Expression: temp, - } - - default: - _, err = iter.ReadAny() // eat up the unused fields - if err != nil { - return nil, err - } - } - } - return q, nil -} diff --git a/experimental/spec/example/query.go b/experimental/spec/example/query.go deleted file mode 100644 index 47e3b1037..000000000 --- a/experimental/spec/example/query.go +++ /dev/null @@ -1,55 +0,0 @@ -package example - -import ( - "fmt" - - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - "github.com/grafana/grafana-plugin-sdk-go/experimental/spec" -) - -// Supported expression types -// +enum -type QueryType string - -const ( - // Math query type - QueryTypeMath QueryType = "math" - - // Reduce query type - QueryTypeReduce QueryType = "reduce" - - // Reduce query type - QueryTypeResample QueryType = "resample" -) - -type ExpressionQuery interface { - ExpressionQueryType() QueryType - Variables() []string -} - -var _ spec.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) - -type QueyHandler struct{} - -// ReadQuery implements query.TypedQueryHandler. -func (*QueyHandler) ParseQuery( - // Properties that have been parsed off the same node - common spec.CommonQueryProperties, - // An iterator with context for the full node (include common values) - iter *jsoniter.Iterator, -) (ExpressionQuery, error) { - qt := QueryType(common.QueryType) - switch qt { - case QueryTypeMath: - return readMathQuery(iter) - - case QueryTypeReduce: - q := &ReduceQuery{} - err := iter.ReadVal(q) - return q, err - - case QueryTypeResample: - return nil, nil - } - return nil, fmt.Errorf("unknown query type") -} diff --git a/experimental/spec/example/query.request.examples.json b/experimental/spec/example/query.request.examples.json deleted file mode 100644 index 871d54dc8..000000000 --- a/experimental/spec/example/query.request.examples.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "from": "now-1h", - "to": "now", - "queries": [ - { - "refId": "A", - "queryType": "math", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A + 10" - }, - { - "refId": "B", - "queryType": "math", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A - $B" - }, - { - "refId": "C", - "queryType": "reduce", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A", - "reducer": "max", - "settings": { - "mode": "dropNN" - } - } - ] -} \ No newline at end of file diff --git a/experimental/spec/example/query.request.schema.json b/experimental/spec/example/query.request.schema.json deleted file mode 100644 index a9a381720..000000000 --- a/experimental/spec/example/query.request.schema.json +++ /dev/null @@ -1,420 +0,0 @@ -{ - "type": "object", - "required": [ - "queries" - ], - "properties": { - "$schema": { - "description": "helper", - "type": "string" - }, - "debug": { - "type": "boolean" - }, - "from": { - "description": "From Start time in epoch timestamps in milliseconds or relative using Grafana time units.", - "type": "string" - }, - "queries": { - "type": "array", - "items": { - "description": "Datasource request model", - "type": "object", - "oneOf": [ - { - "type": "object", - "required": [ - "expression", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "expression": { - "description": "General math expression", - "type": "string", - "minLength": 1, - "examples": [ - "$A + 1", - "$A/$B" - ] - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "intervalMs": { - "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", - "type": "number" - }, - "maxDataPoints": { - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", - "type": "integer" - }, - "queryType": { - "type": "string", - "pattern": "^math$" - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string" - }, - "to": { - "description": "To is the end time of the query.", - "type": "string" - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - }, - { - "type": "object", - "required": [ - "expression", - "reducer", - "settings", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "expression": { - "description": "Reference to other query results", - "type": "string" - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "intervalMs": { - "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", - "type": "number" - }, - "maxDataPoints": { - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", - "type": "integer" - }, - "queryType": { - "type": "string", - "pattern": "^reduce$" - }, - "reducer": { - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "x-enum-description": { - "mean": "The mean", - "sum": "The sum" - } - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "settings": { - "description": "Reducer Options", - "type": "object", - "required": [ - "mode" - ], - "properties": { - "mode": { - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "x-enum-description": { - "dropNN": "Drop non-numbers", - "replaceNN": "Replace non-numbers" - } - }, - "replaceWithValue": { - "description": "Only valid when mode is replace", - "type": "number" - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string" - }, - "to": { - "description": "To is the end time of the query.", - "type": "string" - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - }, - { - "description": "QueryType = resample", - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "downsampler": { - "description": "The reducer", - "type": "string" - }, - "expression": { - "description": "The math expression", - "type": "string" - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "intervalMs": { - "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", - "type": "number" - }, - "loadedDimensions": { - "type": "object", - "additionalProperties": true, - "x-grafana-type": "data.DataFrame" - }, - "maxDataPoints": { - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", - "type": "integer" - }, - "queryType": { - "type": "string", - "pattern": "^resample$" - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string" - }, - "to": { - "description": "To is the end time of the query.", - "type": "string" - } - }, - "additionalProperties": false - }, - "upsampler": { - "description": "The reducer", - "type": "string" - }, - "window": { - "description": "A time duration string", - "type": "string" - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - } - ], - "$schema": "https://json-schema.org/draft-04/schema" - } - }, - "to": { - "description": "To end time in epoch timestamps in milliseconds or relative using Grafana time units.", - "type": "string" - } - }, - "additionalProperties": false -} \ No newline at end of file diff --git a/experimental/spec/example/query.schema.json b/experimental/spec/example/query.schema.json deleted file mode 100644 index dcb5bff32..000000000 --- a/experimental/spec/example/query.schema.json +++ /dev/null @@ -1,369 +0,0 @@ -{ - "description": "object saved in dashboard/alert", - "type": "object", - "oneOf": [ - { - "type": "object", - "required": [ - "expression", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "expression": { - "description": "General math expression", - "type": "string", - "minLength": 1, - "examples": [ - "$A + 1", - "$A/$B" - ] - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "queryType": { - "type": "string", - "pattern": "^math$" - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string" - }, - "to": { - "description": "To is the end time of the query.", - "type": "string" - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - }, - { - "type": "object", - "required": [ - "expression", - "reducer", - "settings", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "expression": { - "description": "Reference to other query results", - "type": "string" - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "queryType": { - "type": "string", - "pattern": "^reduce$" - }, - "reducer": { - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "x-enum-description": { - "mean": "The mean", - "sum": "The sum" - } - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "settings": { - "description": "Reducer Options", - "type": "object", - "required": [ - "mode" - ], - "properties": { - "mode": { - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "x-enum-description": { - "dropNN": "Drop non-numbers", - "replaceNN": "Replace non-numbers" - } - }, - "replaceWithValue": { - "description": "Only valid when mode is replace", - "type": "number" - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string" - }, - "to": { - "description": "To is the end time of the query.", - "type": "string" - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - }, - { - "description": "QueryType = resample", - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "downsampler": { - "description": "The reducer", - "type": "string" - }, - "expression": { - "description": "The math expression", - "type": "string" - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "loadedDimensions": { - "type": "object", - "additionalProperties": true, - "x-grafana-type": "data.DataFrame" - }, - "queryType": { - "type": "string", - "pattern": "^resample$" - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string" - }, - "to": { - "description": "To is the end time of the query.", - "type": "string" - } - }, - "additionalProperties": false - }, - "upsampler": { - "description": "The reducer", - "type": "string" - }, - "window": { - "description": "A time duration string", - "type": "string" - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - } - ], - "$schema": "https://json-schema.org/draft-04/schema" -} \ No newline at end of file diff --git a/experimental/spec/example/query.types.json b/experimental/spec/example/query.types.json deleted file mode 100644 index a8d296764..000000000 --- a/experimental/spec/example/query.types.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "kind": "QueryTypeDefinitionList", - "apiVersion": "query.grafana.app/v0alpha1", - "metadata": { - "resourceVersion": "1708548629808" - }, - "items": [ - { - "metadata": { - "name": "math", - "resourceVersion": "1708817676338", - "creationTimestamp": "2024-02-21T20:50:29Z" - }, - "spec": { - "discriminators": [ - { - "field": "queryType", - "value": "math" - } - ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", - "additionalProperties": false, - "properties": { - "expression": { - "description": "General math expression", - "examples": [ - "$A + 1", - "$A/$B" - ], - "minLength": 1, - "type": "string" - } - }, - "required": [ - "expression" - ], - "type": "object" - }, - "examples": [ - { - "name": "constant addition", - "saveModel": { - "expression": "$A + 10" - } - }, - { - "name": "math with two queries", - "saveModel": { - "expression": "$A - $B" - } - } - ] - } - }, - { - "metadata": { - "name": "reduce", - "resourceVersion": "1708876267448", - "creationTimestamp": "2024-02-21T20:50:29Z" - }, - "spec": { - "discriminators": [ - { - "field": "queryType", - "value": "reduce" - } - ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", - "properties": { - "expression": { - "type": "string", - "description": "Reference to other query results" - }, - "reducer": { - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", - "x-enum-description": { - "mean": "The mean", - "sum": "The sum" - } - }, - "settings": { - "properties": { - "mode": { - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "x-enum-description": { - "dropNN": "Drop non-numbers", - "replaceNN": "Replace non-numbers" - } - }, - "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "mode" - ], - "description": "Reducer Options" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "reducer", - "settings" - ] - }, - "examples": [ - { - "name": "get max value", - "saveModel": { - "expression": "$A", - "reducer": "max", - "settings": { - "mode": "dropNN" - } - } - } - ] - } - }, - { - "metadata": { - "name": "resample", - "resourceVersion": "1708548629808", - "creationTimestamp": "2024-02-21T20:50:29Z" - }, - "spec": { - "discriminators": [ - { - "field": "queryType", - "value": "resample" - } - ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", - "additionalProperties": false, - "description": "QueryType = resample", - "properties": { - "downsampler": { - "description": "The reducer", - "type": "string" - }, - "expression": { - "description": "The math expression", - "type": "string" - }, - "loadedDimensions": { - "additionalProperties": true, - "type": "object", - "x-grafana-type": "data.DataFrame" - }, - "upsampler": { - "description": "The reducer", - "type": "string" - }, - "window": { - "description": "A time duration string", - "type": "string" - } - }, - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions" - ], - "type": "object" - } - } - } - ] -} \ No newline at end of file diff --git a/experimental/spec/example/query_test.go b/experimental/spec/example/query_test.go deleted file mode 100644 index e298852c5..000000000 --- a/experimental/spec/example/query_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package example - -import ( - "reflect" - "testing" - - "github.com/grafana/grafana-plugin-sdk-go/experimental/spec" - "github.com/stretchr/testify/require" -) - -func TestQueryTypeDefinitions(t *testing.T) { - builder, err := spec.NewSchemaBuilder(spec.BuilderOptions{ - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/spec/example", - CodePath: "./", - // We need to identify the enum fields explicitly :( - // *AND* have the +enum common for this to work - Enums: []reflect.Type{ - reflect.TypeOf(ReducerSum), // pick an example value (not the root) - reflect.TypeOf(ReduceModeDrop), // pick an example value (not the root) - }, - }) - require.NoError(t, err) - err = builder.AddQueries(spec.QueryTypeInfo{ - Discriminators: spec.NewDiscriminators("queryType", QueryTypeMath), - GoType: reflect.TypeOf(&MathQuery{}), - Examples: []spec.QueryExample{ - { - Name: "constant addition", - SaveModel: MathQuery{ - Expression: "$A + 10", - }, - }, - { - Name: "math with two queries", - SaveModel: MathQuery{ - Expression: "$A - $B", - }, - }, - }, - }, - spec.QueryTypeInfo{ - Discriminators: spec.NewDiscriminators("queryType", QueryTypeReduce), - GoType: reflect.TypeOf(&ReduceQuery{}), - Examples: []spec.QueryExample{ - { - Name: "get max value", - SaveModel: ReduceQuery{ - Expression: "$A", - Reducer: ReducerMax, - Settings: ReduceSettings{ - Mode: ReduceModeDrop, - }, - }, - }, - }, - }, - spec.QueryTypeInfo{ - Discriminators: spec.NewDiscriminators("queryType", QueryTypeResample), - GoType: reflect.TypeOf(&ResampleQuery{}), - }) - require.NoError(t, err) - - _ = builder.UpdateQueryDefinition(t, "./") -} diff --git a/experimental/spec/example/reduce.go b/experimental/spec/example/reduce.go deleted file mode 100644 index 3893cc06c..000000000 --- a/experimental/spec/example/reduce.go +++ /dev/null @@ -1,57 +0,0 @@ -package example - -var _ ExpressionQuery = (*ReduceQuery)(nil) - -type ReduceQuery struct { - // Reference to other query results - Expression string `json:"expression"` - - // The reducer - Reducer ReducerID `json:"reducer"` - - // Reducer Options - Settings ReduceSettings `json:"settings"` -} - -func (*ReduceQuery) ExpressionQueryType() QueryType { - return QueryTypeReduce -} - -func (q *ReduceQuery) Variables() []string { - return []string{q.Expression} -} - -type ReduceSettings struct { - // Non-number reduce behavior - Mode ReduceMode `json:"mode"` - - // Only valid when mode is replace - ReplaceWithValue *float64 `json:"replaceWithValue,omitempty"` -} - -// The reducer function -// +enum -type ReducerID string - -const ( - // The sum - ReducerSum ReducerID = "sum" - // The mean - ReducerMean ReducerID = "mean" - ReducerMin ReducerID = "min" - ReducerMax ReducerID = "max" - ReducerCount ReducerID = "count" - ReducerLast ReducerID = "last" -) - -// Non-Number behavior mode -// +enum -type ReduceMode string - -const ( - // Drop non-numbers - ReduceModeDrop ReduceMode = "dropNN" - - // Replace non-numbers - ReduceModeReplace ReduceMode = "replaceNN" -) diff --git a/experimental/spec/example/resample.go b/experimental/spec/example/resample.go deleted file mode 100644 index 77f27b1c6..000000000 --- a/experimental/spec/example/resample.go +++ /dev/null @@ -1,28 +0,0 @@ -package example - -import "github.com/grafana/grafana-plugin-sdk-go/data" - -// QueryType = resample -type ResampleQuery struct { - // The math expression - Expression string `json:"expression"` - - // A time duration string - Window string `json:"window"` - - // The reducer - Downsampler string `json:"downsampler"` - - // The reducer - Upsampler string `json:"upsampler"` - - LoadedDimensions *data.Frame `json:"loadedDimensions"` -} - -func (*ResampleQuery) ExpressionQueryType() QueryType { - return QueryTypeReduce -} - -func (q *ResampleQuery) Variables() []string { - return []string{q.Expression} -} diff --git a/experimental/spec/k8s.go b/experimental/spec/k8s.go deleted file mode 100644 index 30c5bdf7e..000000000 --- a/experimental/spec/k8s.go +++ /dev/null @@ -1,52 +0,0 @@ -package spec - -// ObjectMeta is a struct that aims to "look" like a real kubernetes object when -// written to JSON, however it does not require the pile of dependencies -// This is really an internal helper until we decide which dependencies make sense -// to require within the SDK -type ObjectMeta struct { - // The name is for k8s and description, but not used in the schema - Name string `json:"name,omitempty"` - // Changes indicate that *something * changed - ResourceVersion string `json:"resourceVersion,omitempty"` - // Timestamp - CreationTimestamp string `json:"creationTimestamp,omitempty"` -} - -// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition -type QueryTypeDefinition struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` - - Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` -} - -// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types -// For simple data sources, there may be only a single query type, however when multiple types -// exist they must be clearly specified with distinct discriminator field+value pairs -type QueryTypeDefinitionList struct { - Kind string `json:"kind"` // "QueryTypeDefinitionList", - APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", - - ObjectMeta `json:"metadata,omitempty"` - - Items []QueryTypeDefinition `json:"items"` -} - -// SettingsDefinition is a kubernetes shaped object that represents a single query definition -type SettingsDefinition struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` - - Spec SettingsDefinitionSpec `json:"spec,omitempty"` -} - -// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types -// For simple data sources, there may be only a single query type, however when multiple types -// exist they must be clearly specified with distinct discriminator field+value pairs -type SettingsDefinitionList struct { - Kind string `json:"kind"` // "SettingsDefinitionList", - APIVersion string `json:"apiVersion"` // "??.common.grafana.app/v0alpha1", - - ObjectMeta `json:"metadata,omitempty"` - - Items []SettingsDefinition `json:"items"` -} diff --git a/experimental/spec/query.go b/experimental/spec/query.go deleted file mode 100644 index de5a32eb5..000000000 --- a/experimental/spec/query.go +++ /dev/null @@ -1,152 +0,0 @@ -package spec - -import ( - "embed" - "encoding/json" - "fmt" - - "github.com/grafana/grafana-plugin-sdk-go/data" -) - -type DiscriminatorFieldValue struct { - // DiscriminatorField is the field used to link behavior to this specific - // query type. It is typically "queryType", but can be another field if necessary - Field string `json:"field"` - - // The discriminator value - Value string `json:"value"` -} - -// using any since this will often be enumerations -func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { - if len(keyvals)%2 != 0 { - panic("values must be even") - } - dis := []DiscriminatorFieldValue{} - for i := 0; i < len(keyvals); i += 2 { - dis = append(dis, DiscriminatorFieldValue{ - Field: fmt.Sprintf("%v", keyvals[i]), - Value: fmt.Sprintf("%v", keyvals[i+1]), - }) - } - return dis -} - -type QueryTypeDefinitionSpec struct { - // Multiple schemas can be defined using discriminators - Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` - - // Describe whe the query type is for - Description string `json:"description,omitempty"` - - // The query schema represents the properties that can be sent to the API - // In many cases, this may be the same properties that are saved in a dashboard - // In the case where the save model is different, we must also specify a save model - QuerySchema any `json:"querySchema"` - - // The save model defines properties that can be saved into dashboard or similar - // These values are processed by frontend components and then sent to the api - // When specified, this schema will be used to validate saved objects rather than - // the query schema - SaveModel any `json:"saveModel,omitempty"` - - // Examples (include a wrapper) ideally a template! - Examples []QueryExample `json:"examples,omitempty"` - - // Changelog defines the changed from the previous version - // All changes in the same version *must* be backwards compatible - // Only notable changes will be shown here, for the full version history see git! - Changelog []string `json:"changelog,omitempty"` -} - -type QueryExample struct { - // Version identifier or empty if only one exists - Name string `json:"name,omitempty"` - - // An example value saved that can be saved in a dashboard - SaveModel any `json:"saveModel,omitempty"` -} - -type CommonQueryProperties struct { - // RefID is the unique identifier of the query, set by the frontend call. - RefID string `json:"refId,omitempty"` - - // Optionally define expected query result behavior - ResultAssertions *ResultAssertions `json:"resultAssertions,omitempty"` - - // TimeRange represents the query range - // NOTE: unlike generic /ds/query, we can now send explicit time values in each query - // NOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly - TimeRange *TimeRange `json:"timeRange,omitempty"` - - // The datasource - Datasource *DataSourceRef `json:"datasource,omitempty"` - - // Deprecated -- use datasource ref instead - DatasourceID int64 `json:"datasourceId,omitempty"` - - // QueryType is an optional identifier for the type of query. - // It can be used to distinguish different types of queries. - QueryType string `json:"queryType,omitempty"` - - // MaxDataPoints is the maximum number of data points that should be returned from a time series query. - // NOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated - // from the number of pixels visible in a visualization - MaxDataPoints int64 `json:"maxDataPoints,omitempty"` - - // Interval is the suggested duration between time points in a time series query. - // NOTE: the values for intervalMs is not saved in the query model. It is typically calculated - // from the interval required to fill a pixels in the visualization - IntervalMS float64 `json:"intervalMs,omitempty"` - - // true if query is disabled (ie should not be returned to the dashboard) - // NOTE: this does not always imply that the query should not be executed since - // the results from a hidden query may be used as the input to other queries (SSE etc) - Hide bool `json:"hide,omitempty"` -} - -type DataSourceRef struct { - // The datasource plugin type - Type string `json:"type"` - - // Datasource UID - UID string `json:"uid"` - - // ?? the datasource API version? (just version, not the group? type | apiVersion?) -} - -// TimeRange represents a time range for a query and is a property of DataQuery. -type TimeRange struct { - // From is the start time of the query. - From string `json:"from"` - - // To is the end time of the query. - To string `json:"to"` -} - -// ResultAssertions define the expected response shape and query behavior. This is useful to -// enforce behavior over time. The assertions are passed to the query engine and can be used -// to fail queries *before* returning them to a client (select * from bigquery!) -type ResultAssertions struct { - // Type asserts that the frame matches a known type structure. - Type data.FrameType `json:"type,omitempty"` - - // TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane - // contract documentation https://grafana.github.io/dataplane/contract/. - TypeVersion data.FrameTypeVersion `json:"typeVersion"` - - // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast - MaxBytes int64 `json:"maxBytes,omitempty"` - - // Maximum frame count - MaxFrames int64 `json:"maxFrames,omitempty"` -} - -//go:embed query.schema.json -var f embed.FS - -// Get the cached feature list (exposed as a k8s resource) -func GetCommonJSONSchema() json.RawMessage { - body, _ := f.ReadFile("query.schema.json") - return body -} diff --git a/experimental/spec/query.schema.json b/experimental/spec/query.schema.json deleted file mode 100644 index 9ed34f630..000000000 --- a/experimental/spec/query.schema.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-04/schema", - "$id": "https://github.com/grafana/grafana-plugin-sdk-go/experimental/spec/common-query-properties", - "properties": { - "refId": { - "type": "string", - "description": "RefID is the unique identifier of the query, set by the frontend call." - }, - "resultAssertions": { - "properties": { - "type": { - "type": "string", - "description": "Type asserts that the frame matches a known type structure." - }, - "typeVersion": { - "items": { - "type": "integer" - }, - "type": "array", - "maxItems": 2, - "minItems": 2, - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." - }, - "maxBytes": { - "type": "integer", - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" - }, - "maxFrames": { - "type": "integer", - "description": "Maximum frame count" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "typeVersion" - ], - "description": "Optionally define expected query result behavior" - }, - "timeRange": { - "properties": { - "from": { - "type": "string", - "description": "From is the start time of the query." - }, - "to": { - "type": "string", - "description": "To is the end time of the query." - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "from", - "to" - ], - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" - }, - "datasource": { - "properties": { - "type": { - "type": "string", - "description": "The datasource plugin type" - }, - "uid": { - "type": "string", - "description": "Datasource UID" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type", - "uid" - ], - "description": "The datasource" - }, - "queryType": { - "type": "string", - "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." - }, - "maxDataPoints": { - "type": "integer", - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" - }, - "intervalMs": { - "type": "number", - "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" - }, - "hide": { - "type": "boolean", - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" - } - }, - "additionalProperties": false, - "type": "object", - "description": "Query properties shared by all data sources" -} \ No newline at end of file diff --git a/experimental/spec/query_parser.go b/experimental/spec/query_parser.go deleted file mode 100644 index 26461da98..000000000 --- a/experimental/spec/query_parser.go +++ /dev/null @@ -1,322 +0,0 @@ -package spec - -import ( - "encoding/json" - "unsafe" - - "github.com/grafana/grafana-plugin-sdk-go/data/converters" - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - j "github.com/json-iterator/go" -) - -func init() { //nolint:gochecknoinits - jsoniter.RegisterTypeEncoder("spec.GenericDataQuery", &genericQueryCodec{}) - jsoniter.RegisterTypeDecoder("spec.GenericDataQuery", &genericQueryCodec{}) -} - -// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing -type GenericDataQuery struct { - CommonQueryProperties `json:",inline"` - - // Additional Properties (that live at the root) - additional map[string]any `json:"-"` // note this uses custom JSON marshalling -} - -type QueryRequest[Q any] struct { - // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. - // example: now-1h - From string `json:"from,omitempty"` - - // To End time in epoch timestamps in milliseconds or relative using Grafana time units. - // example: now - To string `json:"to,omitempty"` - - // Each item has a - Queries []Q `json:"queries"` - - // required: false - Debug bool `json:"debug,omitempty"` -} - -// Generic query parser pattern. -type TypedQueryParser[Q any] interface { - // Get the query parser for a query type - // The version is split from the end of the discriminator field - ParseQuery( - // Properties that have been parsed off the same node - common CommonQueryProperties, - // An iterator with context for the full node (include common values) - iter *jsoniter.Iterator, - ) (Q, error) -} - -var commonKeys = map[string]bool{ - "refId": true, - "resultAssertions": true, - "timeRange": true, - "datasource": true, - "datasourceId": true, - "queryType": true, - "maxDataPoints": true, - "intervalMs": true, - "hide": true, -} - -var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) - -type GenericQueryParser struct{} - -// ParseQuery implements TypedQueryParser. -func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator) (GenericDataQuery, error) { - q := GenericDataQuery{CommonQueryProperties: common, additional: make(map[string]any)} - field, err := iter.ReadObject() - for field != "" && err == nil { - if !commonKeys[field] { - q.additional[field], err = iter.Read() - if err != nil { - return q, err - } - } - field, err = iter.ReadObject() - } - return q, err -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery. -func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { - if g == nil { - return nil - } - out := new(GenericDataQuery) - jj, err := json.Marshal(g) - if err != nil { - _ = json.Unmarshal(jj, out) - } - return out -} - -func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { - clone := g.DeepCopy() - *out = *clone -} - -// Set allows setting values using key/value pairs -func (g *GenericDataQuery) Set(key string, val any) *GenericDataQuery { - switch key { - case "refId": - g.RefID, _ = val.(string) - case "resultAssertions": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.ResultAssertions) - } - case "timeRange": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.TimeRange) - } - case "datasource": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.Datasource) - } - case "datasourceId": - v, err := converters.JSONValueToInt64.Converter(val) - if err != nil { - g.DatasourceID, _ = v.(int64) - } - case "queryType": - g.QueryType, _ = val.(string) - case "maxDataPoints": - v, err := converters.JSONValueToInt64.Converter(val) - if err != nil { - g.MaxDataPoints, _ = v.(int64) - } - case "intervalMs": - v, err := converters.JSONValueToFloat64.Converter(val) - if err != nil { - g.IntervalMS, _ = v.(float64) - } - case "hide": - g.Hide, _ = val.(bool) - default: - if g.additional == nil { - g.additional = make(map[string]any) - } - g.additional[key] = val - } - return g -} - -func (g *GenericDataQuery) Get(key string) (any, bool) { - switch key { - case "refId": - return g.RefID, true - case "resultAssertions": - return g.ResultAssertions, true - case "timeRange": - return g.TimeRange, true - case "datasource": - return g.Datasource, true - case "datasourceId": - return g.DatasourceID, true - case "queryType": - return g.QueryType, true - case "maxDataPoints": - return g.MaxDataPoints, true - case "intervalMs": - return g.IntervalMS, true - case "hide": - return g.Hide, true - } - v, ok := g.additional[key] - return v, ok -} - -type genericQueryCodec struct{} - -func (codec *genericQueryCodec) IsEmpty(_ unsafe.Pointer) bool { - return false -} - -func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { - q := (*GenericDataQuery)(ptr) - writeQuery(q, stream) -} - -func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { - q := GenericDataQuery{} - err := readQuery(&q, jsoniter.NewIterator(iter)) - if err != nil { - // keep existing iter error if it exists - if iter.Error == nil { - iter.Error = err - } - return - } - *((*GenericDataQuery)(ptr)) = q -} - -// MarshalJSON writes JSON including the common and custom values -func (g GenericDataQuery) MarshalJSON() ([]byte, error) { - cfg := j.ConfigCompatibleWithStandardLibrary - stream := cfg.BorrowStream(nil) - defer cfg.ReturnStream(stream) - - writeQuery(&g, stream) - return append([]byte(nil), stream.Buffer()...), stream.Error -} - -// UnmarshalJSON reads a query from json byte array -func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { - iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, b) - if err != nil { - return err - } - return readQuery(g, iter) -} - -func writeQuery(g *GenericDataQuery, stream *j.Stream) { - q := g.CommonQueryProperties - stream.WriteObjectStart() - stream.WriteObjectField("refId") - stream.WriteVal(g.RefID) - - if q.ResultAssertions != nil { - stream.WriteMore() - stream.WriteObjectField("resultAssertions") - stream.WriteVal(g.ResultAssertions) - } - - if q.TimeRange != nil { - stream.WriteMore() - stream.WriteObjectField("timeRange") - stream.WriteVal(g.TimeRange) - } - - if q.Datasource != nil { - stream.WriteMore() - stream.WriteObjectField("datasource") - stream.WriteVal(g.Datasource) - } - - if q.DatasourceID > 0 { - stream.WriteMore() - stream.WriteObjectField("datasourceId") - stream.WriteVal(g.DatasourceID) - } - - if q.QueryType != "" { - stream.WriteMore() - stream.WriteObjectField("queryType") - stream.WriteVal(g.QueryType) - } - - if q.MaxDataPoints > 0 { - stream.WriteMore() - stream.WriteObjectField("maxDataPoints") - stream.WriteVal(g.MaxDataPoints) - } - - if q.IntervalMS > 0 { - stream.WriteMore() - stream.WriteObjectField("intervalMs") - stream.WriteVal(g.IntervalMS) - } - - if q.Hide { - stream.WriteMore() - stream.WriteObjectField("hide") - stream.WriteVal(g.Hide) - } - - // The additional properties - if g.additional != nil { - for k, v := range g.additional { - stream.WriteMore() - stream.WriteObjectField(k) - stream.WriteVal(v) - } - } - stream.WriteObjectEnd() -} - -func readQuery(g *GenericDataQuery, iter *jsoniter.Iterator) error { - var err error - field := "" - for field, err = iter.ReadObject(); field != ""; field, err = iter.ReadObject() { - switch field { - case "refId": - g.RefID, err = iter.ReadString() - case "resultAssertions": - err = iter.ReadVal(&g.ResultAssertions) - case "timeRange": - err = iter.ReadVal(&g.TimeRange) - case "datasource": - err = iter.ReadVal(&g.Datasource) - case "datasourceId": - g.DatasourceID, err = iter.ReadInt64() - case "queryType": - g.QueryType, err = iter.ReadString() - case "maxDataPoints": - g.MaxDataPoints, err = iter.ReadInt64() - case "intervalMs": - g.IntervalMS, err = iter.ReadFloat64() - case "hide": - g.Hide, err = iter.ReadBool() - default: - v, err := iter.Read() - if err != nil { - return err - } - if g.additional == nil { - g.additional = make(map[string]any) - } - g.additional[field] = v - } - if err != nil { - return err - } - } - return err -} diff --git a/experimental/spec/query_test.go b/experimental/spec/query_test.go deleted file mode 100644 index 5fdbec661..000000000 --- a/experimental/spec/query_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package spec - -import ( - "encoding/json" - "fmt" - "os" - "testing" - - "github.com/invopop/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCommonSupport(t *testing.T) { - r := new(jsonschema.Reflector) - r.DoNotReference = true - err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/spec", "./") - require.NoError(t, err) - - query := r.Reflect(&CommonQueryProperties{}) - query.Version = draft04 // used by kube-openapi - query.Description = "Query properties shared by all data sources" - - // Write the map of values ignored by the common parser - fmt.Printf("var commonKeys = map[string]bool{\n") - for pair := query.Properties.Oldest(); pair != nil; pair = pair.Next() { - fmt.Printf(" \"%s\": true,\n", pair.Key) - } - fmt.Printf("}\n") - - // // Hide this old property - query.Properties.Delete("datasourceId") - out, err := json.MarshalIndent(query, "", " ") - require.NoError(t, err) - - update := false - outfile := "query.schema.json" - body, err := os.ReadFile(outfile) - if err == nil { - if !assert.JSONEq(t, string(out), string(body)) { - update = true - } - } else { - update = true - } - if update { - err = os.WriteFile(outfile, out, 0600) - require.NoError(t, err, "error writing file") - } -} diff --git a/experimental/spec/settings.go b/experimental/spec/settings.go deleted file mode 100644 index 30e092d37..000000000 --- a/experimental/spec/settings.go +++ /dev/null @@ -1,23 +0,0 @@ -package spec - -type SettingsDefinitionSpec struct { - // Multiple schemas can be defined using discriminators - Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` - - // Describe whe the query type is for - Description string `json:"description,omitempty"` - - // The query schema represents the properties that can be sent to the API - // In many cases, this may be the same properties that are saved in a dashboard - // In the case where the save model is different, we must also specify a save model - JSONDataSchema any `json:"jsonDataSchema"` - - // JSON schema defining the properties needed in secure json - // NOTE these must all be string fields - SecureJSONSchema any `json:"secureJsonSchema"` - - // Changelog defines the changed from the previous version - // All changes in the same version *must* be backwards compatible - // Only notable changes will be shown here, for the full version history see git! - Changelog []string `json:"changelog,omitempty"` -} diff --git a/experimental/testdata/folder.golden.txt b/experimental/testdata/folder.golden.txt index 5a2d86ffb..7abe047c3 100644 --- a/experimental/testdata/folder.golden.txt +++ b/experimental/testdata/folder.golden.txt @@ -29,4 +29,4 @@ Dimensions: 2 Fields by 20 Rows ====== TEST DATA RESPONSE (arrow base64) ====== -FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAIAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAA/QAAAAAAAABYAQAAAAAAAAAAAAAAAAAAWAEAAAAAAABUAAAAAAAAALABAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA8QAAAPUAAAD9AAAAAAAAAFJFQURNRS5tZGFjdGlvbnNhdXRoY2xpZW50ZGF0YXNvdXJjZXRlc3RlMmVlcnJvcnNvdXJjZWZlYXR1cmV0b2dnbGVzZmlsZWluZm8uZ29maWxlaW5mb190ZXN0LmdvZnJhbWVfc29ydGVyLmdvZnJhbWVfc29ydGVyX3Rlc3QuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlci5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyX3Rlc3QuZ29odHRwX2xvZ2dlcm1hY3Jvc21vY2tvYXV0aHRva2VucmV0cmlldmVycmVzdF9jbGllbnQuZ29zcGVjdGVzdGRhdGEAAAAAAAAAAAAAAAkAAAASAAAAGwAAACQAAAAtAAAANgAAADYAAAA2AAAANgAAADYAAAA2AAAANgAAAD8AAABIAAAAUQAAAFoAAABaAAAAYwAAAGwAAAAAAAAAZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5AAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAAEAAEAAADYAQAAAAAAAOAAAAAAAAAAIAIAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAPgBAABBUlJPVzE= +FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAKAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAAAQEAAAAAAABgAQAAAAAAAAAAAAAAAAAAYAEAAAAAAABUAAAAAAAAALgBAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA6wAAAPkAAAABAQAAAAAAAFJFQURNRS5tZGFjdGlvbnNhdXRoY2xpZW50ZGF0YXNvdXJjZXRlc3RlMmVlcnJvcnNvdXJjZWZlYXR1cmV0b2dnbGVzZmlsZWluZm8uZ29maWxlaW5mb190ZXN0LmdvZnJhbWVfc29ydGVyLmdvZnJhbWVfc29ydGVyX3Rlc3QuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlci5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyX3Rlc3QuZ29odHRwX2xvZ2dlcm1hY3Jvc21vY2tvYXV0aHRva2VucmV0cmlldmVycmVzb3VyY2VyZXN0X2NsaWVudC5nb3Rlc3RkYXRhAAAAAAAAAAAAAAAAAAAACQAAABIAAAAbAAAAJAAAAC0AAAA2AAAANgAAADYAAAA2AAAANgAAADYAAAA2AAAAPwAAAEgAAABRAAAAWgAAAGMAAABjAAAAbAAAAAAAAABkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnkAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAANgBAAAAAAAA4AAAAAAAAAAoAgAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAuAAAAAMAAABMAAAAKAAAAAQAAADA/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAOD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAP///wgAAABQAAAARAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwidHlwZVZlcnNpb24iOlswLDBdLCJwYXRoU2VwYXJhdG9yIjoiLyJ9AAAAAAQAAABtZXRhAAAAAAIAAAB4AAAABAAAAKL///8UAAAAPAAAADwAAAAAAAAFOAAAAAEAAAAEAAAAkP///wgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA+AEAAEFSUk9XMQ== From e71794177e6c2d4df45058689e262d2ee7eaa77b Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 14:14:11 -0800 Subject: [PATCH 27/71] refactor (again) --- experimental/resource/metaV1.go | 14 + experimental/resource/panel.go | 24 + experimental/resource/query.go | 176 +++++++ experimental/resource/query.schema.json | 107 +++++ experimental/resource/query_parser.go | 322 +++++++++++++ experimental/resource/query_test.go | 56 +++ experimental/resource/schemabuilder/enums.go | 151 ++++++ .../resource/schemabuilder/enums_test.go | 22 + .../resource/schemabuilder/example/math.go | 46 ++ .../resource/schemabuilder/example/query.go | 55 +++ .../example/query.request.examples.json | 31 ++ .../example/query.request.schema.json | 449 ++++++++++++++++++ .../schemabuilder/example/query.schema.json | 398 ++++++++++++++++ .../schemabuilder/example/query.types.json | 193 ++++++++ .../schemabuilder/example/query_test.go | 65 +++ .../resource/schemabuilder/example/reduce.go | 57 +++ .../schemabuilder/example/resample.go | 28 ++ .../resource/schemabuilder/examples.go | 41 ++ .../resource/schemabuilder/reflector.go | 371 +++++++++++++++ experimental/resource/schemabuilder/schema.go | 181 +++++++ experimental/resource/settings.go | 42 ++ 21 files changed, 2829 insertions(+) create mode 100644 experimental/resource/metaV1.go create mode 100644 experimental/resource/panel.go create mode 100644 experimental/resource/query.go create mode 100644 experimental/resource/query.schema.json create mode 100644 experimental/resource/query_parser.go create mode 100644 experimental/resource/query_test.go create mode 100644 experimental/resource/schemabuilder/enums.go create mode 100644 experimental/resource/schemabuilder/enums_test.go create mode 100644 experimental/resource/schemabuilder/example/math.go create mode 100644 experimental/resource/schemabuilder/example/query.go create mode 100644 experimental/resource/schemabuilder/example/query.request.examples.json create mode 100644 experimental/resource/schemabuilder/example/query.request.schema.json create mode 100644 experimental/resource/schemabuilder/example/query.schema.json create mode 100644 experimental/resource/schemabuilder/example/query.types.json create mode 100644 experimental/resource/schemabuilder/example/query_test.go create mode 100644 experimental/resource/schemabuilder/example/reduce.go create mode 100644 experimental/resource/schemabuilder/example/resample.go create mode 100644 experimental/resource/schemabuilder/examples.go create mode 100644 experimental/resource/schemabuilder/reflector.go create mode 100644 experimental/resource/schemabuilder/schema.go create mode 100644 experimental/resource/settings.go diff --git a/experimental/resource/metaV1.go b/experimental/resource/metaV1.go new file mode 100644 index 000000000..a519383c5 --- /dev/null +++ b/experimental/resource/metaV1.go @@ -0,0 +1,14 @@ +package resource + +// ObjectMeta is a struct that aims to "look" like a real kubernetes object when +// written to JSON, however it does not require the pile of dependencies +// This is really an internal helper until we decide which dependencies make sense +// to require within the SDK +type ObjectMeta struct { + // The name is for k8s and description, but not used in the schema + Name string `json:"name,omitempty"` + // Changes indicate that *something * changed + ResourceVersion string `json:"resourceVersion,omitempty"` + // Timestamp + CreationTimestamp string `json:"creationTimestamp,omitempty"` +} diff --git a/experimental/resource/panel.go b/experimental/resource/panel.go new file mode 100644 index 000000000..1ad11bc03 --- /dev/null +++ b/experimental/resource/panel.go @@ -0,0 +1,24 @@ +package resource + +type PseudoPanel[Q any] struct { + // Numeric panel id + ID int `json:"id,omitempty"` + + // The panel plugin type + Type string `json:"type"` + + // The panel title + Title string `json:"title,omitempty"` + + // Options depend on the panel type + Options map[string]any `json:"options,omitempty"` + + // FieldConfig values depend on the panel type + FieldConfig map[string]any `json:"fieldConfig,omitempty"` + + // This should no longer be necessary since each target has the datasource reference + Datasource DataSourceRef `json:"datasource,omitempty"` + + // The query targets + Targets []Q `json:"targets"` +} diff --git a/experimental/resource/query.go b/experimental/resource/query.go new file mode 100644 index 000000000..32ef5fe1f --- /dev/null +++ b/experimental/resource/query.go @@ -0,0 +1,176 @@ +package resource + +import ( + "embed" + "fmt" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition +type QueryTypeDefinition struct { + ObjectMeta ObjectMeta `json:"metadata,omitempty"` + + Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` +} + +// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types +// For simple data sources, there may be only a single query type, however when multiple types +// exist they must be clearly specified with distinct discriminator field+value pairs +type QueryTypeDefinitionList struct { + Kind string `json:"kind"` // "QueryTypeDefinitionList", + APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", + + ObjectMeta `json:"metadata,omitempty"` + + Items []QueryTypeDefinition `json:"items"` +} + +type QueryTypeDefinitionSpec struct { + // Multiple schemas can be defined using discriminators + Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // The query schema represents the properties that can be sent to the API + // In many cases, this may be the same properties that are saved in a dashboard + // In the case where the save model is different, we must also specify a save model + QuerySchema any `json:"querySchema"` + + // The save model defines properties that can be saved into dashboard or similar + // These values are processed by frontend components and then sent to the api + // When specified, this schema will be used to validate saved objects rather than + // the query schema + SaveModel any `json:"saveModel,omitempty"` + + // Examples (include a wrapper) ideally a template! + Examples []QueryExample `json:"examples,omitempty"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} + +type QueryExample struct { + // Version identifier or empty if only one exists + Name string `json:"name,omitempty"` + + // An example value saved that can be saved in a dashboard + SaveModel any `json:"saveModel,omitempty"` +} + +type CommonQueryProperties struct { + // RefID is the unique identifier of the query, set by the frontend call. + RefID string `json:"refId,omitempty"` + + // Optionally define expected query result behavior + ResultAssertions *ResultAssertions `json:"resultAssertions,omitempty"` + + // TimeRange represents the query range + // NOTE: unlike generic /ds/query, we can now send explicit time values in each query + // NOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly + TimeRange *TimeRange `json:"timeRange,omitempty"` + + // The datasource + Datasource *DataSourceRef `json:"datasource,omitempty"` + + // Deprecated -- use datasource ref instead + DatasourceID int64 `json:"datasourceId,omitempty"` + + // QueryType is an optional identifier for the type of query. + // It can be used to distinguish different types of queries. + QueryType string `json:"queryType,omitempty"` + + // MaxDataPoints is the maximum number of data points that should be returned from a time series query. + // NOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated + // from the number of pixels visible in a visualization + MaxDataPoints int64 `json:"maxDataPoints,omitempty"` + + // Interval is the suggested duration between time points in a time series query. + // NOTE: the values for intervalMs is not saved in the query model. It is typically calculated + // from the interval required to fill a pixels in the visualization + IntervalMS float64 `json:"intervalMs,omitempty"` + + // true if query is disabled (ie should not be returned to the dashboard) + // NOTE: this does not always imply that the query should not be executed since + // the results from a hidden query may be used as the input to other queries (SSE etc) + Hide bool `json:"hide,omitempty"` +} + +type DataSourceRef struct { + // The datasource plugin type + Type string `json:"type"` + + // Datasource UID + UID string `json:"uid"` + + // ?? the datasource API version? (just version, not the group? type | apiVersion?) +} + +// TimeRange represents a time range for a query and is a property of DataQuery. +type TimeRange struct { + // From is the start time of the query. + From string `json:"from" jsonschema:"example=now-1h"` + + // To is the end time of the query. + To string `json:"to" jsonschema:"example=now"` +} + +// ResultAssertions define the expected response shape and query behavior. This is useful to +// enforce behavior over time. The assertions are passed to the query engine and can be used +// to fail queries *before* returning them to a client (select * from bigquery!) +type ResultAssertions struct { + // Type asserts that the frame matches a known type structure. + Type data.FrameType `json:"type,omitempty" jsonschema:"example=timeseries-wide,example=timeseries-long"` + + // TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane + // contract documentation https://grafana.github.io/dataplane/contract/. + TypeVersion data.FrameTypeVersion `json:"typeVersion"` + + // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast + MaxBytes int64 `json:"maxBytes,omitempty"` + + // Maximum frame count + MaxFrames int64 `json:"maxFrames,omitempty"` +} + +type DiscriminatorFieldValue struct { + // DiscriminatorField is the field used to link behavior to this specific + // query type. It is typically "queryType", but can be another field if necessary + Field string `json:"field"` + + // The discriminator value + Value string `json:"value"` +} + +// using any since this will often be enumerations +func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { + if len(keyvals)%2 != 0 { + panic("values must be even") + } + dis := []DiscriminatorFieldValue{} + for i := 0; i < len(keyvals); i += 2 { + dis = append(dis, DiscriminatorFieldValue{ + Field: fmt.Sprintf("%v", keyvals[i]), + Value: fmt.Sprintf("%v", keyvals[i+1]), + }) + } + return dis +} + +//go:embed query.schema.json +var f embed.FS + +// Get the cached feature list (exposed as a k8s resource) +func CommonQueryPropertiesSchema() (*spec.Schema, error) { + body, err := f.ReadFile("query.schema.json") + if err != nil { + return nil, err + } + s := &spec.Schema{} + err = s.UnmarshalJSON(body) + return s, err +} diff --git a/experimental/resource/query.schema.json b/experimental/resource/query.schema.json new file mode 100644 index 000000000..5dafdb75f --- /dev/null +++ b/experimental/resource/query.schema.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema", + "properties": { + "refId": { + "type": "string", + "description": "RefID is the unique identifier of the query, set by the frontend call." + }, + "resultAssertions": { + "properties": { + "type": { + "type": "string", + "description": "Type asserts that the frame matches a known type structure.", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2, + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." + }, + "maxBytes": { + "type": "integer", + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" + }, + "maxFrames": { + "type": "integer", + "description": "Maximum frame count" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "typeVersion" + ], + "description": "Optionally define expected query result behavior" + }, + "timeRange": { + "properties": { + "from": { + "type": "string", + "description": "From is the start time of the query.", + "examples": [ + "now-1h" + ] + }, + "to": { + "type": "string", + "description": "To is the end time of the query.", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "from", + "to" + ], + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" + }, + "datasource": { + "properties": { + "type": { + "type": "string", + "description": "The datasource plugin type" + }, + "uid": { + "type": "string", + "description": "Datasource UID" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type", + "uid" + ], + "description": "The datasource" + }, + "queryType": { + "type": "string", + "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." + }, + "maxDataPoints": { + "type": "integer", + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" + }, + "intervalMs": { + "type": "number", + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" + }, + "hide": { + "type": "boolean", + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" + } + }, + "additionalProperties": false, + "type": "object", + "description": "Query properties shared by all data sources" +} \ No newline at end of file diff --git a/experimental/resource/query_parser.go b/experimental/resource/query_parser.go new file mode 100644 index 000000000..f97bf2b61 --- /dev/null +++ b/experimental/resource/query_parser.go @@ -0,0 +1,322 @@ +package resource + +import ( + "encoding/json" + "unsafe" + + "github.com/grafana/grafana-plugin-sdk-go/data/converters" + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + j "github.com/json-iterator/go" +) + +func init() { //nolint:gochecknoinits + jsoniter.RegisterTypeEncoder("resource.GenericDataQuery", &genericQueryCodec{}) + jsoniter.RegisterTypeDecoder("resource.GenericDataQuery", &genericQueryCodec{}) +} + +// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing +type GenericDataQuery struct { + CommonQueryProperties `json:",inline"` + + // Additional Properties (that live at the root) + additional map[string]any `json:"-"` // note this uses custom JSON marshalling +} + +type QueryRequest[Q any] struct { + // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. + // example: now-1h + From string `json:"from,omitempty"` + + // To End time in epoch timestamps in milliseconds or relative using Grafana time units. + // example: now + To string `json:"to,omitempty"` + + // Each item has a + Queries []Q `json:"queries"` + + // required: false + Debug bool `json:"debug,omitempty"` +} + +// Generic query parser pattern. +type TypedQueryParser[Q any] interface { + // Get the query parser for a query type + // The version is split from the end of the discriminator field + ParseQuery( + // Properties that have been parsed off the same node + common CommonQueryProperties, + // An iterator with context for the full node (include common values) + iter *jsoniter.Iterator, + ) (Q, error) +} + +var commonKeys = map[string]bool{ + "refId": true, + "resultAssertions": true, + "timeRange": true, + "datasource": true, + "datasourceId": true, + "queryType": true, + "maxDataPoints": true, + "intervalMs": true, + "hide": true, +} + +var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) + +type GenericQueryParser struct{} + +// ParseQuery implements TypedQueryParser. +func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator) (GenericDataQuery, error) { + q := GenericDataQuery{CommonQueryProperties: common, additional: make(map[string]any)} + field, err := iter.ReadObject() + for field != "" && err == nil { + if !commonKeys[field] { + q.additional[field], err = iter.Read() + if err != nil { + return q, err + } + } + field, err = iter.ReadObject() + } + return q, err +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery. +func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { + if g == nil { + return nil + } + out := new(GenericDataQuery) + jj, err := json.Marshal(g) + if err != nil { + _ = json.Unmarshal(jj, out) + } + return out +} + +func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { + clone := g.DeepCopy() + *out = *clone +} + +// Set allows setting values using key/value pairs +func (g *GenericDataQuery) Set(key string, val any) *GenericDataQuery { + switch key { + case "refId": + g.RefID, _ = val.(string) + case "resultAssertions": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.ResultAssertions) + } + case "timeRange": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.TimeRange) + } + case "datasource": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.Datasource) + } + case "datasourceId": + v, err := converters.JSONValueToInt64.Converter(val) + if err != nil { + g.DatasourceID, _ = v.(int64) + } + case "queryType": + g.QueryType, _ = val.(string) + case "maxDataPoints": + v, err := converters.JSONValueToInt64.Converter(val) + if err != nil { + g.MaxDataPoints, _ = v.(int64) + } + case "intervalMs": + v, err := converters.JSONValueToFloat64.Converter(val) + if err != nil { + g.IntervalMS, _ = v.(float64) + } + case "hide": + g.Hide, _ = val.(bool) + default: + if g.additional == nil { + g.additional = make(map[string]any) + } + g.additional[key] = val + } + return g +} + +func (g *GenericDataQuery) Get(key string) (any, bool) { + switch key { + case "refId": + return g.RefID, true + case "resultAssertions": + return g.ResultAssertions, true + case "timeRange": + return g.TimeRange, true + case "datasource": + return g.Datasource, true + case "datasourceId": + return g.DatasourceID, true + case "queryType": + return g.QueryType, true + case "maxDataPoints": + return g.MaxDataPoints, true + case "intervalMs": + return g.IntervalMS, true + case "hide": + return g.Hide, true + } + v, ok := g.additional[key] + return v, ok +} + +type genericQueryCodec struct{} + +func (codec *genericQueryCodec) IsEmpty(_ unsafe.Pointer) bool { + return false +} + +func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { + q := (*GenericDataQuery)(ptr) + writeQuery(q, stream) +} + +func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { + q := GenericDataQuery{} + err := readQuery(&q, jsoniter.NewIterator(iter)) + if err != nil { + // keep existing iter error if it exists + if iter.Error == nil { + iter.Error = err + } + return + } + *((*GenericDataQuery)(ptr)) = q +} + +// MarshalJSON writes JSON including the common and custom values +func (g GenericDataQuery) MarshalJSON() ([]byte, error) { + cfg := j.ConfigCompatibleWithStandardLibrary + stream := cfg.BorrowStream(nil) + defer cfg.ReturnStream(stream) + + writeQuery(&g, stream) + return append([]byte(nil), stream.Buffer()...), stream.Error +} + +// UnmarshalJSON reads a query from json byte array +func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { + iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, b) + if err != nil { + return err + } + return readQuery(g, iter) +} + +func writeQuery(g *GenericDataQuery, stream *j.Stream) { + q := g.CommonQueryProperties + stream.WriteObjectStart() + stream.WriteObjectField("refId") + stream.WriteVal(g.RefID) + + if q.ResultAssertions != nil { + stream.WriteMore() + stream.WriteObjectField("resultAssertions") + stream.WriteVal(g.ResultAssertions) + } + + if q.TimeRange != nil { + stream.WriteMore() + stream.WriteObjectField("timeRange") + stream.WriteVal(g.TimeRange) + } + + if q.Datasource != nil { + stream.WriteMore() + stream.WriteObjectField("datasource") + stream.WriteVal(g.Datasource) + } + + if q.DatasourceID > 0 { + stream.WriteMore() + stream.WriteObjectField("datasourceId") + stream.WriteVal(g.DatasourceID) + } + + if q.QueryType != "" { + stream.WriteMore() + stream.WriteObjectField("queryType") + stream.WriteVal(g.QueryType) + } + + if q.MaxDataPoints > 0 { + stream.WriteMore() + stream.WriteObjectField("maxDataPoints") + stream.WriteVal(g.MaxDataPoints) + } + + if q.IntervalMS > 0 { + stream.WriteMore() + stream.WriteObjectField("intervalMs") + stream.WriteVal(g.IntervalMS) + } + + if q.Hide { + stream.WriteMore() + stream.WriteObjectField("hide") + stream.WriteVal(g.Hide) + } + + // The additional properties + if g.additional != nil { + for k, v := range g.additional { + stream.WriteMore() + stream.WriteObjectField(k) + stream.WriteVal(v) + } + } + stream.WriteObjectEnd() +} + +func readQuery(g *GenericDataQuery, iter *jsoniter.Iterator) error { + var err error + field := "" + for field, err = iter.ReadObject(); field != ""; field, err = iter.ReadObject() { + switch field { + case "refId": + g.RefID, err = iter.ReadString() + case "resultAssertions": + err = iter.ReadVal(&g.ResultAssertions) + case "timeRange": + err = iter.ReadVal(&g.TimeRange) + case "datasource": + err = iter.ReadVal(&g.Datasource) + case "datasourceId": + g.DatasourceID, err = iter.ReadInt64() + case "queryType": + g.QueryType, err = iter.ReadString() + case "maxDataPoints": + g.MaxDataPoints, err = iter.ReadInt64() + case "intervalMs": + g.IntervalMS, err = iter.ReadFloat64() + case "hide": + g.Hide, err = iter.ReadBool() + default: + v, err := iter.Read() + if err != nil { + return err + } + if g.additional == nil { + g.additional = make(map[string]any) + } + g.additional[field] = v + } + if err != nil { + return err + } + } + return err +} diff --git a/experimental/resource/query_test.go b/experimental/resource/query_test.go new file mode 100644 index 000000000..d3b9272d3 --- /dev/null +++ b/experimental/resource/query_test.go @@ -0,0 +1,56 @@ +package resource + +import ( + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCommonQueryProperties(t *testing.T) { + r := new(jsonschema.Reflector) + r.DoNotReference = true + err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/resource", "./") + require.NoError(t, err) + + query := r.Reflect(&CommonQueryProperties{}) + query.ID = "" + query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi + query.Description = "Query properties shared by all data sources" + + // Write the map of values ignored by the common parser + fmt.Printf("var commonKeys = map[string]bool{\n") + for pair := query.Properties.Oldest(); pair != nil; pair = pair.Next() { + fmt.Printf(" \"%s\": true,\n", pair.Key) + } + fmt.Printf("}\n") + + // // Hide this old property + query.Properties.Delete("datasourceId") + out, err := json.MarshalIndent(query, "", " ") + require.NoError(t, err) + + update := false + outfile := "query.schema.json" + body, err := os.ReadFile(outfile) + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true + } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0600) + require.NoError(t, err, "error writing file") + } + + // Make sure the embedded schema is loadable + schema, err := CommonQueryPropertiesSchema() + require.NoError(t, err) + require.Equal(t, 8, len(schema.Properties)) +} diff --git a/experimental/resource/schemabuilder/enums.go b/experimental/resource/schemabuilder/enums.go new file mode 100644 index 000000000..7d9d3c931 --- /dev/null +++ b/experimental/resource/schemabuilder/enums.go @@ -0,0 +1,151 @@ +package schemabuilder + +import ( + "fmt" + "io/fs" + gopath "path" + "path/filepath" + "regexp" + "strings" + + "go/ast" + "go/doc" + "go/parser" + "go/token" + + "github.com/invopop/jsonschema" +) + +type EnumValue struct { + Value string + Comment string +} + +type EnumField struct { + Package string + Name string + Comment string + Values []EnumValue +} + +func findEnumFields(base, path string) ([]EnumField, error) { + fset := token.NewFileSet() + dict := make(map[string][]*ast.Package) + err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + d, err := parser.ParseDir(fset, path, nil, parser.ParseComments) + if err != nil { + return err + } + for _, v := range d { + // paths may have multiple packages, like for tests + k := gopath.Join(base, path) + dict[k] = append(dict[k], v) + } + } + return nil + }) + if err != nil { + return nil, err + } + + fields := make([]EnumField, 0) + field := &EnumField{} + dp := &doc.Package{} + + for pkg, p := range dict { + for _, f := range p { + gtxt := "" + typ := "" + ast.Inspect(f, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.TypeSpec: + typ = x.Name.String() + if !ast.IsExported(typ) { + typ = "" + } else { + txt := x.Doc.Text() + if txt == "" && gtxt != "" { + txt = gtxt + gtxt = "" + } + txt = strings.TrimSpace(dp.Synopsis(txt)) + if strings.HasSuffix(txt, "+enum") { + fields = append(fields, EnumField{ + Package: pkg, + Name: typ, + Comment: strings.TrimSpace(strings.TrimSuffix(txt, "+enum")), + }) + field = &fields[len(fields)-1] + } + } + case *ast.ValueSpec: + txt := x.Doc.Text() + if txt == "" { + txt = x.Comment.Text() + } + if typ == field.Name { + for _, n := range x.Names { + if ast.IsExported(n.String()) { + v, ok := x.Values[0].(*ast.BasicLit) + if ok { + val := strings.TrimPrefix(v.Value, `"`) + val = strings.TrimSuffix(val, `"`) + txt = strings.TrimSpace(txt) + field.Values = append(field.Values, EnumValue{ + Value: val, + Comment: txt, + }) + } + } + } + } + case *ast.GenDecl: + // remember for the next type + gtxt = x.Doc.Text() + } + return true + }) + } + } + + return fields, nil +} + +// whitespaceRegex is the regex for consecutive whitespaces. +var whitespaceRegex = regexp.MustCompile(`\s+`) + +func UpdateEnumDescriptions(s *jsonschema.Schema) { + if len(s.Enum) > 0 && s.Extras != nil { + extra, ok := s.Extras["x-enum-description"] + if !ok { + return + } + + lookup, ok := extra.(map[string]string) + if !ok { + return + } + + lines := []string{} + if s.Description != "" { + lines = append(lines, s.Description, "\n") + } + lines = append(lines, "Possible enum values:") + for _, v := range s.Enum { + c := lookup[v.(string)] + c = whitespaceRegex.ReplaceAllString(c, " ") + lines = append(lines, fmt.Sprintf(" - `%q` %s", v, c)) + } + + s.Description = strings.Join(lines, "\n") + return + } + + for pair := s.Properties.Oldest(); pair != nil; pair = pair.Next() { + UpdateEnumDescriptions(pair.Value) + } +} diff --git a/experimental/resource/schemabuilder/enums_test.go b/experimental/resource/schemabuilder/enums_test.go new file mode 100644 index 000000000..745dd5b59 --- /dev/null +++ b/experimental/resource/schemabuilder/enums_test.go @@ -0,0 +1,22 @@ +package schemabuilder + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFindEnums(t *testing.T) { + fields, err := findEnumFields( + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder", + "./example") + require.NoError(t, err) + + out, err := json.MarshalIndent(fields, "", " ") + require.NoError(t, err) + fmt.Printf("%s", string(out)) + + require.Equal(t, 3, len(fields)) +} diff --git a/experimental/resource/schemabuilder/example/math.go b/experimental/resource/schemabuilder/example/math.go new file mode 100644 index 000000000..0ef5d43c6 --- /dev/null +++ b/experimental/resource/schemabuilder/example/math.go @@ -0,0 +1,46 @@ +package example + +import "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + +var _ ExpressionQuery = (*MathQuery)(nil) + +type MathQuery struct { + // General math expression + Expression string `json:"expression" jsonschema:"minLength=1,example=$A + 1,example=$A/$B"` + + // Parsed from the expression + variables []string `json:"-"` +} + +func (*MathQuery) ExpressionQueryType() QueryType { + return QueryTypeMath +} + +func (q *MathQuery) Variables() []string { + return q.variables +} + +func readMathQuery(iter *jsoniter.Iterator) (*MathQuery, error) { + var q *MathQuery + var err error + fname := "" + for fname, err = iter.ReadObject(); fname != "" && err == nil; fname, err = iter.ReadObject() { + switch fname { + case "expression": + temp, err := iter.ReadString() + if err != nil { + return q, err + } + q = &MathQuery{ + Expression: temp, + } + + default: + _, err = iter.ReadAny() // eat up the unused fields + if err != nil { + return nil, err + } + } + } + return q, nil +} diff --git a/experimental/resource/schemabuilder/example/query.go b/experimental/resource/schemabuilder/example/query.go new file mode 100644 index 000000000..b1bfc5340 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query.go @@ -0,0 +1,55 @@ +package example + +import ( + "fmt" + + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" +) + +// Supported expression types +// +enum +type QueryType string + +const ( + // Math query type + QueryTypeMath QueryType = "math" + + // Reduce query type + QueryTypeReduce QueryType = "reduce" + + // Reduce query type + QueryTypeResample QueryType = "resample" +) + +type ExpressionQuery interface { + ExpressionQueryType() QueryType + Variables() []string +} + +var _ resource.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) + +type QueyHandler struct{} + +// ReadQuery implements query.TypedQueryHandler. +func (*QueyHandler) ParseQuery( + // Properties that have been parsed off the same node + common resource.CommonQueryProperties, + // An iterator with context for the full node (include common values) + iter *jsoniter.Iterator, +) (ExpressionQuery, error) { + qt := QueryType(common.QueryType) + switch qt { + case QueryTypeMath: + return readMathQuery(iter) + + case QueryTypeReduce: + q := &ReduceQuery{} + err := iter.ReadVal(q) + return q, err + + case QueryTypeResample: + return nil, nil + } + return nil, fmt.Errorf("unknown query type") +} diff --git a/experimental/resource/schemabuilder/example/query.request.examples.json b/experimental/resource/schemabuilder/example/query.request.examples.json new file mode 100644 index 000000000..871d54dc8 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query.request.examples.json @@ -0,0 +1,31 @@ +{ + "from": "now-1h", + "to": "now", + "queries": [ + { + "refId": "A", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A + 10" + }, + { + "refId": "B", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A - $B" + }, + { + "refId": "C", + "queryType": "reduce", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + ] +} \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.request.schema.json b/experimental/resource/schemabuilder/example/query.request.schema.json new file mode 100644 index 000000000..5fac0d320 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query.request.schema.json @@ -0,0 +1,449 @@ +{ + "type": "object", + "required": [ + "queries" + ], + "properties": { + "$schema": { + "description": "helper", + "type": "string" + }, + "debug": { + "type": "boolean" + }, + "from": { + "description": "From Start time in epoch timestamps in milliseconds or relative using Grafana time units.", + "type": "string" + }, + "queries": { + "type": "array", + "items": { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "expression": { + "description": "General math expression", + "type": "string", + "minLength": 1, + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "intervalMs": { + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", + "type": "number" + }, + "maxDataPoints": { + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", + "type": "integer" + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + }, + { + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "expression": { + "description": "Reference to other query results", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "intervalMs": { + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", + "type": "number" + }, + "maxDataPoints": { + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", + "type": "integer" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "reducer": { + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" + } + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "settings": { + "description": "Reducer Options", + "type": "object", + "required": [ + "mode" + ], + "properties": { + "mode": { + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "x-enum-description": { + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" + } + }, + "replaceWithValue": { + "description": "Only valid when mode is replace", + "type": "number" + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + }, + { + "description": "QueryType = resample", + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "downsampler": { + "description": "The reducer", + "type": "string" + }, + "expression": { + "description": "The math expression", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "intervalMs": { + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization", + "type": "number" + }, + "loadedDimensions": { + "type": "object", + "additionalProperties": true, + "x-grafana-type": "data.DataFrame" + }, + "maxDataPoints": { + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization", + "type": "integer" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + }, + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "A time duration string", + "type": "string" + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + } + ], + "$schema": "https://json-schema.org/draft-04/schema" + } + }, + "to": { + "description": "To end time in epoch timestamps in milliseconds or relative using Grafana time units.", + "type": "string" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.schema.json b/experimental/resource/schemabuilder/example/query.schema.json new file mode 100644 index 000000000..651de2530 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query.schema.json @@ -0,0 +1,398 @@ +{ + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "expression": { + "description": "General math expression", + "type": "string", + "minLength": 1, + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + }, + { + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "expression": { + "description": "Reference to other query results", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "reducer": { + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" + } + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "settings": { + "description": "Reducer Options", + "type": "object", + "required": [ + "mode" + ], + "properties": { + "mode": { + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "x-enum-description": { + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" + } + }, + "replaceWithValue": { + "description": "Only valid when mode is replace", + "type": "number" + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + }, + { + "description": "QueryType = resample", + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type", + "uid" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "downsampler": { + "description": "The reducer", + "type": "string" + }, + "expression": { + "description": "The math expression", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "loadedDimensions": { + "type": "object", + "additionalProperties": true, + "x-grafana-type": "data.DataFrame" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + }, + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "A time duration string", + "type": "string" + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + } + ], + "$schema": "https://json-schema.org/draft-04/schema" +} \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.types.json b/experimental/resource/schemabuilder/example/query.types.json new file mode 100644 index 000000000..a8d296764 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query.types.json @@ -0,0 +1,193 @@ +{ + "kind": "QueryTypeDefinitionList", + "apiVersion": "query.grafana.app/v0alpha1", + "metadata": { + "resourceVersion": "1708548629808" + }, + "items": [ + { + "metadata": { + "name": "math", + "resourceVersion": "1708817676338", + "creationTimestamp": "2024-02-21T20:50:29Z" + }, + "spec": { + "discriminators": [ + { + "field": "queryType", + "value": "math" + } + ], + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", + "additionalProperties": false, + "properties": { + "expression": { + "description": "General math expression", + "examples": [ + "$A + 1", + "$A/$B" + ], + "minLength": 1, + "type": "string" + } + }, + "required": [ + "expression" + ], + "type": "object" + }, + "examples": [ + { + "name": "constant addition", + "saveModel": { + "expression": "$A + 10" + } + }, + { + "name": "math with two queries", + "saveModel": { + "expression": "$A - $B" + } + } + ] + } + }, + { + "metadata": { + "name": "reduce", + "resourceVersion": "1708876267448", + "creationTimestamp": "2024-02-21T20:50:29Z" + }, + "spec": { + "discriminators": [ + { + "field": "queryType", + "value": "reduce" + } + ], + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", + "properties": { + "expression": { + "type": "string", + "description": "Reference to other query results" + }, + "reducer": { + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" + } + }, + "settings": { + "properties": { + "mode": { + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "x-enum-description": { + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" + } + }, + "replaceWithValue": { + "type": "number", + "description": "Only valid when mode is replace" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "mode" + ], + "description": "Reducer Options" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "expression", + "reducer", + "settings" + ] + }, + "examples": [ + { + "name": "get max value", + "saveModel": { + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + } + ] + } + }, + { + "metadata": { + "name": "resample", + "resourceVersion": "1708548629808", + "creationTimestamp": "2024-02-21T20:50:29Z" + }, + "spec": { + "discriminators": [ + { + "field": "queryType", + "value": "resample" + } + ], + "querySchema": { + "$schema": "https://json-schema.org/draft-04/schema", + "additionalProperties": false, + "description": "QueryType = resample", + "properties": { + "downsampler": { + "description": "The reducer", + "type": "string" + }, + "expression": { + "description": "The math expression", + "type": "string" + }, + "loadedDimensions": { + "additionalProperties": true, + "type": "object", + "x-grafana-type": "data.DataFrame" + }, + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "A time duration string", + "type": "string" + } + }, + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions" + ], + "type": "object" + } + } + } + ] +} \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query_test.go b/experimental/resource/schemabuilder/example/query_test.go new file mode 100644 index 000000000..12ccb4c37 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query_test.go @@ -0,0 +1,65 @@ +package example + +import ( + "reflect" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder" + "github.com/stretchr/testify/require" +) + +func TestQueryTypeDefinitions(t *testing.T) { + builder, err := schemabuilder.NewSchemaBuilder(schemabuilder.BuilderOptions{ + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder/example", + CodePath: "./", + // We need to identify the enum fields explicitly :( + // *AND* have the +enum common for this to work + Enums: []reflect.Type{ + reflect.TypeOf(ReducerSum), // pick an example value (not the root) + reflect.TypeOf(ReduceModeDrop), // pick an example value (not the root) + }, + }) + require.NoError(t, err) + err = builder.AddQueries(schemabuilder.QueryTypeInfo{ + Discriminators: resource.NewDiscriminators("queryType", QueryTypeMath), + GoType: reflect.TypeOf(&MathQuery{}), + Examples: []resource.QueryExample{ + { + Name: "constant addition", + SaveModel: MathQuery{ + Expression: "$A + 10", + }, + }, + { + Name: "math with two queries", + SaveModel: MathQuery{ + Expression: "$A - $B", + }, + }, + }, + }, + schemabuilder.QueryTypeInfo{ + Discriminators: resource.NewDiscriminators("queryType", QueryTypeReduce), + GoType: reflect.TypeOf(&ReduceQuery{}), + Examples: []resource.QueryExample{ + { + Name: "get max value", + SaveModel: ReduceQuery{ + Expression: "$A", + Reducer: ReducerMax, + Settings: ReduceSettings{ + Mode: ReduceModeDrop, + }, + }, + }, + }, + }, + schemabuilder.QueryTypeInfo{ + Discriminators: resource.NewDiscriminators("queryType", QueryTypeResample), + GoType: reflect.TypeOf(&ResampleQuery{}), + }) + require.NoError(t, err) + + _ = builder.UpdateQueryDefinition(t, "./") +} diff --git a/experimental/resource/schemabuilder/example/reduce.go b/experimental/resource/schemabuilder/example/reduce.go new file mode 100644 index 000000000..3893cc06c --- /dev/null +++ b/experimental/resource/schemabuilder/example/reduce.go @@ -0,0 +1,57 @@ +package example + +var _ ExpressionQuery = (*ReduceQuery)(nil) + +type ReduceQuery struct { + // Reference to other query results + Expression string `json:"expression"` + + // The reducer + Reducer ReducerID `json:"reducer"` + + // Reducer Options + Settings ReduceSettings `json:"settings"` +} + +func (*ReduceQuery) ExpressionQueryType() QueryType { + return QueryTypeReduce +} + +func (q *ReduceQuery) Variables() []string { + return []string{q.Expression} +} + +type ReduceSettings struct { + // Non-number reduce behavior + Mode ReduceMode `json:"mode"` + + // Only valid when mode is replace + ReplaceWithValue *float64 `json:"replaceWithValue,omitempty"` +} + +// The reducer function +// +enum +type ReducerID string + +const ( + // The sum + ReducerSum ReducerID = "sum" + // The mean + ReducerMean ReducerID = "mean" + ReducerMin ReducerID = "min" + ReducerMax ReducerID = "max" + ReducerCount ReducerID = "count" + ReducerLast ReducerID = "last" +) + +// Non-Number behavior mode +// +enum +type ReduceMode string + +const ( + // Drop non-numbers + ReduceModeDrop ReduceMode = "dropNN" + + // Replace non-numbers + ReduceModeReplace ReduceMode = "replaceNN" +) diff --git a/experimental/resource/schemabuilder/example/resample.go b/experimental/resource/schemabuilder/example/resample.go new file mode 100644 index 000000000..77f27b1c6 --- /dev/null +++ b/experimental/resource/schemabuilder/example/resample.go @@ -0,0 +1,28 @@ +package example + +import "github.com/grafana/grafana-plugin-sdk-go/data" + +// QueryType = resample +type ResampleQuery struct { + // The math expression + Expression string `json:"expression"` + + // A time duration string + Window string `json:"window"` + + // The reducer + Downsampler string `json:"downsampler"` + + // The reducer + Upsampler string `json:"upsampler"` + + LoadedDimensions *data.Frame `json:"loadedDimensions"` +} + +func (*ResampleQuery) ExpressionQueryType() QueryType { + return QueryTypeReduce +} + +func (q *ResampleQuery) Variables() []string { + return []string{q.Expression} +} diff --git a/experimental/resource/schemabuilder/examples.go b/experimental/resource/schemabuilder/examples.go new file mode 100644 index 000000000..50f4fe743 --- /dev/null +++ b/experimental/resource/schemabuilder/examples.go @@ -0,0 +1,41 @@ +package schemabuilder + +import ( + "fmt" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" +) + +func GetExampleQueries(defs resource.QueryTypeDefinitionList) (resource.QueryRequest[resource.GenericDataQuery], error) { + rsp := resource.QueryRequest[resource.GenericDataQuery]{ + From: "now-1h", + To: "now", + Queries: []resource.GenericDataQuery{}, + } + + for _, def := range defs.Items { + for _, sample := range def.Spec.Examples { + if sample.SaveModel != nil { + q, err := asGenericDataQuery(sample.SaveModel) + if err != nil { + return rsp, fmt.Errorf("invalid sample save query [%s], in %s // %w", + sample.Name, def.ObjectMeta.Name, err) + } + q.RefID = string(rune('A' + len(rsp.Queries))) + for _, dis := range def.Spec.Discriminators { + _ = q.Set(dis.Field, dis.Value) + } + + if q.MaxDataPoints < 1 { + q.MaxDataPoints = 1000 + } + if q.IntervalMS < 1 { + q.IntervalMS = 5 + } + + rsp.Queries = append(rsp.Queries, *q) + } + } + } + return rsp, nil +} diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go new file mode 100644 index 000000000..f8bb2abeb --- /dev/null +++ b/experimental/resource/schemabuilder/reflector.go @@ -0,0 +1,371 @@ +package schemabuilder + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/kube-openapi/pkg/validation/strfmt" + "k8s.io/kube-openapi/pkg/validation/validate" +) + +// SchemaBuilder is a helper function that can be used by +// backend build processes to produce static schema definitions +// This is not intended as runtime code, and is not the only way to +// produce a schema (we may also want/need to use typescript as the source) +type Builder struct { + opts BuilderOptions + reflector *jsonschema.Reflector // Needed to use comments + query []resource.QueryTypeDefinition + setting []resource.SettingsDefinition +} + +type BuilderOptions struct { + // The plugin type ID used in the DataSourceRef type property + PluginID []string + + // ex "github.com/grafana/github-datasource/pkg/models" + BasePackage string + + // ex "./" + CodePath string + + // explicitly define the enumeration fields + Enums []reflect.Type +} + +type QueryTypeInfo struct { + // The management name + Name string + // Optional discriminators + Discriminators []resource.DiscriminatorFieldValue + // Raw GO type used for reflection + GoType reflect.Type + // Add sample queries + Examples []resource.QueryExample +} + +type SettingTypeInfo struct { + // The management name + Name string + // Optional discriminators + Discriminators []resource.DiscriminatorFieldValue + // Raw GO type used for reflection + GoType reflect.Type + // Map[string]string + SecureGoType reflect.Type +} + +func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { + r := new(jsonschema.Reflector) + r.DoNotReference = true + if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { + return nil, err + } + customMapper := map[reflect.Type]*jsonschema.Schema{ + reflect.TypeOf(data.Frame{}): { + Type: "object", + Extras: map[string]any{ + "x-grafana-type": "data.DataFrame", + }, + AdditionalProperties: jsonschema.TrueSchema, + }, + } + r.Mapper = func(t reflect.Type) *jsonschema.Schema { + return customMapper[t] + } + + if len(opts.Enums) > 0 { + fields, err := findEnumFields(opts.BasePackage, opts.CodePath) + if err != nil { + return nil, err + } + for _, etype := range opts.Enums { + for _, f := range fields { + if f.Name == etype.Name() && f.Package == etype.PkgPath() { + enumValueDescriptions := map[string]string{} + s := &jsonschema.Schema{ + Type: "string", + Extras: map[string]any{ + "x-enum-description": enumValueDescriptions, + }, + } + for _, val := range f.Values { + s.Enum = append(s.Enum, val.Value) + if val.Comment != "" { + enumValueDescriptions[val.Value] = val.Comment + } + } + customMapper[etype] = s + } + } + } + } + + return &Builder{ + opts: opts, + reflector: r, + }, nil +} + +func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { + for _, info := range inputs { + schema := b.reflector.ReflectFromType(info.GoType) + if schema == nil { + return fmt.Errorf("missing schema") + } + + UpdateEnumDescriptions(schema) + + name := info.Name + if name == "" { + for _, dis := range info.Discriminators { + if name != "" { + name += "-" + } + name += dis.Value + } + if name == "" { + return fmt.Errorf("missing name or discriminators") + } + } + + // We need to be careful to only use draft-04 so that this is possible to use + // with kube-openapi + schema.Version = draft04 + schema.ID = "" + schema.Anchor = "" + + b.query = append(b.query, resource.QueryTypeDefinition{ + ObjectMeta: resource.ObjectMeta{ + Name: name, + }, + Spec: resource.QueryTypeDefinitionSpec{ + Discriminators: info.Discriminators, + QuerySchema: schema, + Examples: info.Examples, + }, + }) + } + return nil +} + +func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { + for _, info := range inputs { + name := info.Name + if name == "" { + return fmt.Errorf("missing name") + } + + schema := b.reflector.ReflectFromType(info.GoType) + if schema == nil { + return fmt.Errorf("missing schema") + } + + UpdateEnumDescriptions(schema) + + // used by kube-openapi + schema.Version = draft04 + schema.ID = "" + schema.Anchor = "" + + b.setting = append(b.setting, resource.SettingsDefinition{ + ObjectMeta: resource.ObjectMeta{ + Name: name, + }, + Spec: resource.SettingsDefinitionSpec{ + Discriminators: info.Discriminators, + JSONDataSchema: schema, + }, + }) + } + return nil +} + +// Update the schema definition file +// When placed in `static/schema/query.types.json` folder of a plugin distribution, +// it can be used to advertise various query types +// If the spec contents have changed, the test will fail (but still update the output) +func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.QueryTypeDefinitionList { + t.Helper() + + outfile := filepath.Join(outdir, "query.types.json") + now := time.Now().UTC() + rv := fmt.Sprintf("%d", now.UnixMilli()) + + defs := resource.QueryTypeDefinitionList{} + byName := make(map[string]*resource.QueryTypeDefinition) + body, err := os.ReadFile(outfile) + if err == nil { + err = json.Unmarshal(body, &defs) + if err == nil { + for i, def := range defs.Items { + byName[def.ObjectMeta.Name] = &defs.Items[i] + } + } + } + defs.Kind = "QueryTypeDefinitionList" + defs.APIVersion = "query.grafana.app/v0alpha1" + + // The updated schemas + for _, def := range b.query { + found, ok := byName[def.ObjectMeta.Name] + if !ok { + defs.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) + + defs.Items = append(defs.Items, def) + } else { + var o1, o2 interface{} + b1, _ := json.Marshal(def.Spec) + b2, _ := json.Marshal(found.Spec) + _ = json.Unmarshal(b1, &o1) + _ = json.Unmarshal(b2, &o2) + if !reflect.DeepEqual(o1, o2) { + found.ObjectMeta.ResourceVersion = rv + found.Spec = def.Spec + } + delete(byName, def.ObjectMeta.Name) + } + } + + if defs.ObjectMeta.ResourceVersion == "" { + defs.ObjectMeta.ResourceVersion = rv + } + + if len(byName) > 0 { + require.FailNow(t, "query type removed, manually update (for now)") + } + maybeUpdateFile(t, outfile, defs, body) + + // Update the query save model schema + //------------------------------------ + outfile = filepath.Join(outdir, "query.schema.json") + schema, err := GetQuerySchema(QuerySchemaOptions{ + QueryTypes: defs.Items, + Mode: SchemaTypePanelModel, + }) + require.NoError(t, err) + + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, schema, body) + + // Update the request payload schema + //------------------------------------ + outfile = filepath.Join(outdir, "query.request.schema.json") + schema, err = GetQuerySchema(QuerySchemaOptions{ + QueryTypes: defs.Items, + Mode: SchemaTypeQueryRequest, + }) + require.NoError(t, err) + + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, schema, body) + + // Verify that the example queries actually validate + //------------------------------------ + request, err := GetExampleQueries(defs) + require.NoError(t, err) + + outfile = filepath.Join(outdir, "query.request.examples.json") + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, request, body) + + validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) + result := validator.Validate(request) + if result.HasErrorsOrWarnings() { + body, err = json.MarshalIndent(result, "", " ") + require.NoError(t, err) + fmt.Printf("Validation: %s\n", string(body)) + require.Fail(t, "validation failed") + } + require.True(t, result.MatchCount > 0, "must have some rules") + return defs +} + +// Update the schema definition file +// When placed in `static/schema/query.schema.json` folder of a plugin distribution, +// it can be used to advertise various query types +// If the spec contents have changed, the test will fail (but still update the output) +func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) resource.SettingsDefinitionList { + t.Helper() + + now := time.Now().UTC() + rv := fmt.Sprintf("%d", now.UnixMilli()) + + defs := resource.SettingsDefinitionList{} + byName := make(map[string]*resource.SettingsDefinition) + body, err := os.ReadFile(outfile) + if err == nil { + err = json.Unmarshal(body, &defs) + if err == nil { + for i, def := range defs.Items { + byName[def.ObjectMeta.Name] = &defs.Items[i] + } + } + } + defs.Kind = "SettingsDefinitionList" + defs.APIVersion = "common.grafana.app/v0alpha1" + + // The updated schemas + for _, def := range b.setting { + found, ok := byName[def.ObjectMeta.Name] + if !ok { + defs.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.ResourceVersion = rv + def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) + + defs.Items = append(defs.Items, def) + } else { + var o1, o2 interface{} + b1, _ := json.Marshal(def.Spec) + b2, _ := json.Marshal(found.Spec) + _ = json.Unmarshal(b1, &o1) + _ = json.Unmarshal(b2, &o2) + if !reflect.DeepEqual(o1, o2) { + found.ObjectMeta.ResourceVersion = rv + found.Spec = def.Spec + } + delete(byName, def.ObjectMeta.Name) + } + } + + if defs.ObjectMeta.ResourceVersion == "" { + defs.ObjectMeta.ResourceVersion = rv + } + + if len(byName) > 0 { + require.FailNow(t, "settings type removed, manually update (for now)") + } + return defs +} + +func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { + t.Helper() + + out, err := json.MarshalIndent(value, "", " ") + require.NoError(t, err) + + update := false + if err == nil { + if !assert.JSONEq(t, string(out), string(body)) { + update = true + } + } else { + update = true + } + if update { + err = os.WriteFile(outfile, out, 0600) + require.NoError(t, err, "error writing file") + } +} diff --git a/experimental/resource/schemabuilder/schema.go b/experimental/resource/schemabuilder/schema.go new file mode 100644 index 000000000..304294334 --- /dev/null +++ b/experimental/resource/schemabuilder/schema.go @@ -0,0 +1,181 @@ +package schemabuilder + +import ( + "encoding/json" + "fmt" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +// The k8s compatible jsonschema version +const draft04 = "https://json-schema.org/draft-04/schema" + +// Supported expression types +// +enum +type SchemaType string + +const ( + // Single query target saved in a dashboard/panel/alert JSON + SchemaTypeSaveModel SchemaType = "save" + + // Single query payload included in a query request + SchemaTypeQueryPayload SchemaType = "payload" + + // Pseudo panel model including multiple targets (not mixed) + SchemaTypePanelModel SchemaType = "panel" + + // Query request against a single data source (not mixed) + SchemaTypeQueryRequest SchemaType = "request" +) + +type QuerySchemaOptions struct { + PluginID []string + QueryTypes []resource.QueryTypeDefinition + Mode SchemaType +} + +// Given definitions for a plugin, return a valid spec +func GetQuerySchema(opts QuerySchemaOptions) (*spec.Schema, error) { + isRequest := opts.Mode == SchemaTypeQueryPayload || opts.Mode == SchemaTypeQueryRequest + generic, err := resource.CommonQueryPropertiesSchema() + if err != nil { + return nil, err + } + + ignoreForSave := map[string]bool{"maxDataPoints": true, "intervalMs": true} + common := make(map[string]spec.Schema) + for key, val := range generic.Properties { + if !isRequest && ignoreForSave[key] { + continue // + } + common[key] = val + } + + // The datasource requirement + switch len(opts.PluginID) { + case 0: + case 1: + s := common["datasource"].Properties["type"] + s.Pattern = "xxxx" + default: + if opts.Mode == SchemaTypePanelModel { + return nil, fmt.Errorf("panel model requires pluginId") + } + s := common["datasource"].Properties["type"] + s.Pattern = "yyyyy" + } + + // The types for each query type + queryTypes := []*spec.Schema{} + for _, qt := range opts.QueryTypes { + node, err := asJSONSchema(qt.Spec.QuerySchema) + if err != nil { + return nil, fmt.Errorf("error reading query types schema: %s // %w", qt.ObjectMeta.Name, err) + } + if node == nil { + return nil, fmt.Errorf("missing query schema: %s // %v", qt.ObjectMeta.Name, qt) + } + + // Match all discriminators + for _, d := range qt.Spec.Discriminators { + ds, ok := node.Properties[d.Field] + if !ok { + ds = *spec.StringProperty() + } + ds.Pattern = `^` + d.Value + `$` + node.Properties[d.Field] = ds + node.Required = append(node.Required, d.Field) + } + + queryTypes = append(queryTypes, node) + } + + s := &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Schema: draft04, + Properties: make(map[string]spec.Schema), + }, + } + + // Single node -- just union the global and local properties + if len(queryTypes) == 1 { + s = queryTypes[0] + s.Schema = draft04 + for key, val := range generic.Properties { + _, found := s.Properties[key] + if found { + continue + } + s.Properties[key] = val + } + } else { + for _, qt := range queryTypes { + qt.Required = append(qt.Required, "refId") + + for k, v := range common { + _, found := qt.Properties[k] + if found { + continue + } + qt.Properties[k] = v + } + + s.OneOf = append(s.OneOf, *qt) + } + } + + if isRequest { + s = addRequestWrapper(s) + } + return s, nil +} + +// moves the schema the the query slot in a request +func addRequestWrapper(s *spec.Schema) *spec.Schema { + return &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Required: []string{"queries"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: false}, + Properties: map[string]spec.Schema{ + "from": *spec.StringProperty().WithDescription( + "From Start time in epoch timestamps in milliseconds or relative using Grafana time units."), + "to": *spec.StringProperty().WithDescription( + "To end time in epoch timestamps in milliseconds or relative using Grafana time units."), + "queries": *spec.ArrayProperty(s), + "debug": *spec.BoolProperty(), + "$schema": *spec.StringProperty().WithDescription("helper"), + }, + }, + } +} + +func asJSONSchema(v any) (*spec.Schema, error) { + s, ok := v.(*spec.Schema) + if ok { + return s, nil + } + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + s = &spec.Schema{} + err = json.Unmarshal(b, s) + return s, err +} + +func asGenericDataQuery(v any) (*resource.GenericDataQuery, error) { + s, ok := v.(*resource.GenericDataQuery) + if ok { + return s, nil + } + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + s = &resource.GenericDataQuery{} + err = json.Unmarshal(b, s) + return s, err +} diff --git a/experimental/resource/settings.go b/experimental/resource/settings.go new file mode 100644 index 000000000..4ef621851 --- /dev/null +++ b/experimental/resource/settings.go @@ -0,0 +1,42 @@ +package resource + +// SettingsDefinition is a kubernetes shaped object that represents a single query definition +type SettingsDefinition struct { + ObjectMeta ObjectMeta `json:"metadata,omitempty"` + + Spec SettingsDefinitionSpec `json:"spec,omitempty"` +} + +// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types +// For simple data sources, there may be only a single query type, however when multiple types +// exist they must be clearly specified with distinct discriminator field+value pairs +type SettingsDefinitionList struct { + Kind string `json:"kind"` // "SettingsDefinitionList", + APIVersion string `json:"apiVersion"` // "??.common.grafana.app/v0alpha1", + + ObjectMeta `json:"metadata,omitempty"` + + Items []SettingsDefinition `json:"items"` +} + +type SettingsDefinitionSpec struct { + // Multiple schemas can be defined using discriminators + Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // The query schema represents the properties that can be sent to the API + // In many cases, this may be the same properties that are saved in a dashboard + // In the case where the save model is different, we must also specify a save model + JSONDataSchema any `json:"jsonDataSchema"` + + // JSON schema defining the properties needed in secure json + // NOTE these must all be string fields + SecureJSONSchema any `json:"secureJsonSchema"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} From d9f700ebab860d9810e906887122905069868961 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 15:20:41 -0800 Subject: [PATCH 28/71] now with panel schema --- experimental/resource/panel.go | 2 +- experimental/resource/query.go | 2 +- experimental/resource/query.schema.json | 3 +- .../example/query.request.examples.json | 31 -- .../example/query.request.schema.json | 21 +- .../schemabuilder/example/query.schema.json | 398 ------------------ .../schemabuilder/example/query_test.go | 1 + .../resource/schemabuilder/examples.go | 25 +- .../resource/schemabuilder/reflector.go | 27 +- experimental/resource/schemabuilder/schema.go | 56 ++- 10 files changed, 102 insertions(+), 464 deletions(-) delete mode 100644 experimental/resource/schemabuilder/example/query.request.examples.json delete mode 100644 experimental/resource/schemabuilder/example/query.schema.json diff --git a/experimental/resource/panel.go b/experimental/resource/panel.go index 1ad11bc03..f2a426a2d 100644 --- a/experimental/resource/panel.go +++ b/experimental/resource/panel.go @@ -17,7 +17,7 @@ type PseudoPanel[Q any] struct { FieldConfig map[string]any `json:"fieldConfig,omitempty"` // This should no longer be necessary since each target has the datasource reference - Datasource DataSourceRef `json:"datasource,omitempty"` + Datasource *DataSourceRef `json:"datasource,omitempty"` // The query targets Targets []Q `json:"targets"` diff --git a/experimental/resource/query.go b/experimental/resource/query.go index 32ef5fe1f..8761ffb2b 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -105,7 +105,7 @@ type DataSourceRef struct { Type string `json:"type"` // Datasource UID - UID string `json:"uid"` + UID string `json:"uid,omitempty"` // ?? the datasource API version? (just version, not the group? type | apiVersion?) } diff --git a/experimental/resource/query.schema.json b/experimental/resource/query.schema.json index 5dafdb75f..302b3e8e3 100644 --- a/experimental/resource/query.schema.json +++ b/experimental/resource/query.schema.json @@ -79,8 +79,7 @@ "additionalProperties": false, "type": "object", "required": [ - "type", - "uid" + "type" ], "description": "The datasource" }, diff --git a/experimental/resource/schemabuilder/example/query.request.examples.json b/experimental/resource/schemabuilder/example/query.request.examples.json deleted file mode 100644 index 871d54dc8..000000000 --- a/experimental/resource/schemabuilder/example/query.request.examples.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "from": "now-1h", - "to": "now", - "queries": [ - { - "refId": "A", - "queryType": "math", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A + 10" - }, - { - "refId": "B", - "queryType": "math", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A - $B" - }, - { - "refId": "C", - "queryType": "reduce", - "maxDataPoints": 1000, - "intervalMs": 5, - "expression": "$A", - "reducer": "max", - "settings": { - "mode": "dropNN" - } - } - ] -} \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.request.schema.json b/experimental/resource/schemabuilder/example/query.request.schema.json index 5fac0d320..d6c753ee3 100644 --- a/experimental/resource/schemabuilder/example/query.request.schema.json +++ b/experimental/resource/schemabuilder/example/query.request.schema.json @@ -32,13 +32,13 @@ "description": "The datasource", "type": "object", "required": [ - "type", - "uid" + "type" ], "properties": { "type": { "description": "The datasource plugin type", - "type": "string" + "type": "string", + "pattern": "^__expr__$" }, "uid": { "description": "Datasource UID", @@ -154,13 +154,13 @@ "description": "The datasource", "type": "object", "required": [ - "type", - "uid" + "type" ], "properties": { "type": { "description": "The datasource plugin type", - "type": "string" + "type": "string", + "pattern": "^__expr__$" }, "uid": { "description": "Datasource UID", @@ -316,13 +316,13 @@ "description": "The datasource", "type": "object", "required": [ - "type", - "uid" + "type" ], "properties": { "type": { "description": "The datasource plugin type", - "type": "string" + "type": "string", + "pattern": "^__expr__$" }, "uid": { "description": "Datasource UID", @@ -445,5 +445,6 @@ "type": "string" } }, - "additionalProperties": false + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" } \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.schema.json b/experimental/resource/schemabuilder/example/query.schema.json deleted file mode 100644 index 651de2530..000000000 --- a/experimental/resource/schemabuilder/example/query.schema.json +++ /dev/null @@ -1,398 +0,0 @@ -{ - "type": "object", - "oneOf": [ - { - "type": "object", - "required": [ - "expression", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "expression": { - "description": "General math expression", - "type": "string", - "minLength": 1, - "examples": [ - "$A + 1", - "$A/$B" - ] - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "queryType": { - "type": "string", - "pattern": "^math$" - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string", - "examples": [ - "timeseries-wide", - "timeseries-long" - ] - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string", - "examples": [ - "now-1h" - ] - }, - "to": { - "description": "To is the end time of the query.", - "type": "string", - "examples": [ - "now" - ] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - }, - { - "type": "object", - "required": [ - "expression", - "reducer", - "settings", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "expression": { - "description": "Reference to other query results", - "type": "string" - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "queryType": { - "type": "string", - "pattern": "^reduce$" - }, - "reducer": { - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", - "type": "string", - "enum": [ - "sum", - "mean", - "min", - "max", - "count", - "last" - ], - "x-enum-description": { - "mean": "The mean", - "sum": "The sum" - } - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string", - "examples": [ - "timeseries-wide", - "timeseries-long" - ] - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "settings": { - "description": "Reducer Options", - "type": "object", - "required": [ - "mode" - ], - "properties": { - "mode": { - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "type": "string", - "enum": [ - "dropNN", - "replaceNN" - ], - "x-enum-description": { - "dropNN": "Drop non-numbers", - "replaceNN": "Replace non-numbers" - } - }, - "replaceWithValue": { - "description": "Only valid when mode is replace", - "type": "number" - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string", - "examples": [ - "now-1h" - ] - }, - "to": { - "description": "To is the end time of the query.", - "type": "string", - "examples": [ - "now" - ] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - }, - { - "description": "QueryType = resample", - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions", - "queryType", - "refId" - ], - "properties": { - "datasource": { - "description": "The datasource", - "type": "object", - "required": [ - "type", - "uid" - ], - "properties": { - "type": { - "description": "The datasource plugin type", - "type": "string" - }, - "uid": { - "description": "Datasource UID", - "type": "string" - } - }, - "additionalProperties": false - }, - "downsampler": { - "description": "The reducer", - "type": "string" - }, - "expression": { - "description": "The math expression", - "type": "string" - }, - "hide": { - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", - "type": "boolean" - }, - "loadedDimensions": { - "type": "object", - "additionalProperties": true, - "x-grafana-type": "data.DataFrame" - }, - "queryType": { - "type": "string", - "pattern": "^resample$" - }, - "refId": { - "description": "RefID is the unique identifier of the query, set by the frontend call.", - "type": "string" - }, - "resultAssertions": { - "description": "Optionally define expected query result behavior", - "type": "object", - "required": [ - "typeVersion" - ], - "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, - "maxFrames": { - "description": "Maximum frame count", - "type": "integer" - }, - "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string", - "examples": [ - "timeseries-wide", - "timeseries-long" - ] - }, - "typeVersion": { - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "integer" - } - } - }, - "additionalProperties": false - }, - "timeRange": { - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", - "type": "object", - "required": [ - "from", - "to" - ], - "properties": { - "from": { - "description": "From is the start time of the query.", - "type": "string", - "examples": [ - "now-1h" - ] - }, - "to": { - "description": "To is the end time of the query.", - "type": "string", - "examples": [ - "now" - ] - } - }, - "additionalProperties": false - }, - "upsampler": { - "description": "The reducer", - "type": "string" - }, - "window": { - "description": "A time duration string", - "type": "string" - } - }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" - } - ], - "$schema": "https://json-schema.org/draft-04/schema" -} \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query_test.go b/experimental/resource/schemabuilder/example/query_test.go index 12ccb4c37..c63299a11 100644 --- a/experimental/resource/schemabuilder/example/query_test.go +++ b/experimental/resource/schemabuilder/example/query_test.go @@ -11,6 +11,7 @@ import ( func TestQueryTypeDefinitions(t *testing.T) { builder, err := schemabuilder.NewSchemaBuilder(schemabuilder.BuilderOptions{ + PluginID: []string{"__expr__"}, BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder/example", CodePath: "./", // We need to identify the enum fields explicitly :( diff --git a/experimental/resource/schemabuilder/examples.go b/experimental/resource/schemabuilder/examples.go index 50f4fe743..db857c717 100644 --- a/experimental/resource/schemabuilder/examples.go +++ b/experimental/resource/schemabuilder/examples.go @@ -6,7 +6,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" ) -func GetExampleQueries(defs resource.QueryTypeDefinitionList) (resource.QueryRequest[resource.GenericDataQuery], error) { +func exampleRequest(defs resource.QueryTypeDefinitionList) (resource.QueryRequest[resource.GenericDataQuery], error) { rsp := resource.QueryRequest[resource.GenericDataQuery]{ From: "now-1h", To: "now", @@ -39,3 +39,26 @@ func GetExampleQueries(defs resource.QueryTypeDefinitionList) (resource.QueryReq } return rsp, nil } + +func examplePanelTargets(ds *resource.DataSourceRef, defs resource.QueryTypeDefinitionList) ([]resource.GenericDataQuery, error) { + targets := []resource.GenericDataQuery{} + + for _, def := range defs.Items { + for _, sample := range def.Spec.Examples { + if sample.SaveModel != nil { + q, err := asGenericDataQuery(sample.SaveModel) + if err != nil { + return nil, fmt.Errorf("invalid sample save query [%s], in %s // %w", + sample.Name, def.ObjectMeta.Name, err) + } + q.Datasource = ds + q.RefID = string(rune('A' + len(targets))) + for _, dis := range def.Spec.Discriminators { + _ = q.Set(dis.Field, dis.Value) + } + targets = append(targets, *q) + } + } + } + return targets, nil +} diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index f8bb2abeb..79d85665c 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -66,6 +66,10 @@ type SettingTypeInfo struct { } func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { + if len(opts.PluginID) < 1 { + return nil, fmt.Errorf("missing plugin id") + } + r := new(jsonschema.Reflector) r.DoNotReference = true if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { @@ -250,8 +254,9 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu // Update the query save model schema //------------------------------------ - outfile = filepath.Join(outdir, "query.schema.json") + outfile = filepath.Join(outdir, "query.panel.schema.json") schema, err := GetQuerySchema(QuerySchemaOptions{ + PluginID: b.opts.PluginID, QueryTypes: defs.Items, Mode: SchemaTypePanelModel, }) @@ -260,10 +265,24 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, schema, body) + panel := resource.PseudoPanel[resource.GenericDataQuery]{ + Type: "table", + } + panel.Targets, err = examplePanelTargets(&resource.DataSourceRef{ + Type: b.opts.PluginID[0], + UID: "TheUID", + }, defs) + require.NoError(t, err) + + outfile = filepath.Join(outdir, "query.panel.example.json") + body, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, panel, body) + // Update the request payload schema //------------------------------------ outfile = filepath.Join(outdir, "query.request.schema.json") schema, err = GetQuerySchema(QuerySchemaOptions{ + PluginID: b.opts.PluginID, QueryTypes: defs.Items, Mode: SchemaTypeQueryRequest, }) @@ -272,12 +291,10 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, schema, body) - // Verify that the example queries actually validate - //------------------------------------ - request, err := GetExampleQueries(defs) + request, err := exampleRequest(defs) require.NoError(t, err) - outfile = filepath.Join(outdir, "query.request.examples.json") + outfile = filepath.Join(outdir, "query.request.example.json") body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, request, body) diff --git a/experimental/resource/schemabuilder/schema.go b/experimental/resource/schemabuilder/schema.go index 304294334..395e03c21 100644 --- a/experimental/resource/schemabuilder/schema.go +++ b/experimental/resource/schemabuilder/schema.go @@ -49,21 +49,27 @@ func GetQuerySchema(opts QuerySchemaOptions) (*spec.Schema, error) { if !isRequest && ignoreForSave[key] { continue // } - common[key] = val - } - // The datasource requirement - switch len(opts.PluginID) { - case 0: - case 1: - s := common["datasource"].Properties["type"] - s.Pattern = "xxxx" - default: - if opts.Mode == SchemaTypePanelModel { - return nil, fmt.Errorf("panel model requires pluginId") + if key == "datasource" { + pattern := "" + for _, pid := range opts.PluginID { + if pattern != "" { + pattern += "|" + } + pattern += `^` + pid + `$` + } + if pattern == "" { + if opts.Mode == SchemaTypePanelModel { + return nil, fmt.Errorf("panel model requires pluginId") + } + } else { + t := val.Properties["type"] + t.Pattern = pattern + val.Properties["type"] = t + } } - s := common["datasource"].Properties["type"] - s.Pattern = "yyyyy" + + common[key] = val } // The types for each query type @@ -126,8 +132,11 @@ func GetQuerySchema(opts QuerySchemaOptions) (*spec.Schema, error) { } } - if isRequest { - s = addRequestWrapper(s) + switch opts.Mode { + case SchemaTypeQueryRequest: + return addRequestWrapper(s), nil + case SchemaTypePanelModel: + return addPanelWrapper(s), nil } return s, nil } @@ -136,6 +145,7 @@ func GetQuerySchema(opts QuerySchemaOptions) (*spec.Schema, error) { func addRequestWrapper(s *spec.Schema) *spec.Schema { return &spec.Schema{ SchemaProps: spec.SchemaProps{ + Schema: draft04, Type: []string{"object"}, Required: []string{"queries"}, AdditionalProperties: &spec.SchemaOrBool{Allows: false}, @@ -152,6 +162,22 @@ func addRequestWrapper(s *spec.Schema) *spec.Schema { } } +// Pretends to be a panel object +func addPanelWrapper(s *spec.Schema) *spec.Schema { + return &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Schema: draft04, + Type: []string{"object"}, + Required: []string{"targets", "type"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + Properties: map[string]spec.Schema{ + "type": *spec.StringProperty().WithDescription("the panel type"), + "targets": *spec.ArrayProperty(s), + }, + }, + } +} + func asJSONSchema(v any) (*spec.Schema, error) { s, ok := v.(*spec.Schema) if ok { From ee12be24cbd1f7bee4a09311897ab8b13fb326c0 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 15:20:48 -0800 Subject: [PATCH 29/71] now with panel schema --- .../example/query.panel.example.json | 36 ++ .../example/query.panel.schema.json | 416 ++++++++++++++++++ .../example/query.request.example.json | 31 ++ 3 files changed, 483 insertions(+) create mode 100644 experimental/resource/schemabuilder/example/query.panel.example.json create mode 100644 experimental/resource/schemabuilder/example/query.panel.schema.json create mode 100644 experimental/resource/schemabuilder/example/query.request.example.json diff --git a/experimental/resource/schemabuilder/example/query.panel.example.json b/experimental/resource/schemabuilder/example/query.panel.example.json new file mode 100644 index 000000000..c02694bb2 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query.panel.example.json @@ -0,0 +1,36 @@ +{ + "type": "table", + "targets": [ + { + "refId": "A", + "datasource": { + "type": "__expr__", + "uid": "TheUID" + }, + "queryType": "math", + "expression": "$A + 10" + }, + { + "refId": "B", + "datasource": { + "type": "__expr__", + "uid": "TheUID" + }, + "queryType": "math", + "expression": "$A - $B" + }, + { + "refId": "C", + "datasource": { + "type": "__expr__", + "uid": "TheUID" + }, + "queryType": "reduce", + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + ] +} \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.panel.schema.json b/experimental/resource/schemabuilder/example/query.panel.schema.json new file mode 100644 index 000000000..bc11a3237 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query.panel.schema.json @@ -0,0 +1,416 @@ +{ + "type": "object", + "required": [ + "targets", + "type" + ], + "properties": { + "targets": { + "type": "array", + "items": { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "expression", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string", + "pattern": "^__expr__$" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "expression": { + "description": "General math expression", + "type": "string", + "minLength": 1, + "examples": [ + "$A + 1", + "$A/$B" + ] + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "queryType": { + "type": "string", + "pattern": "^math$" + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + }, + { + "type": "object", + "required": [ + "expression", + "reducer", + "settings", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string", + "pattern": "^__expr__$" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "expression": { + "description": "Reference to other query results", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "queryType": { + "type": "string", + "pattern": "^reduce$" + }, + "reducer": { + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", + "type": "string", + "enum": [ + "sum", + "mean", + "min", + "max", + "count", + "last" + ], + "x-enum-description": { + "mean": "The mean", + "sum": "The sum" + } + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "settings": { + "description": "Reducer Options", + "type": "object", + "required": [ + "mode" + ], + "properties": { + "mode": { + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", + "type": "string", + "enum": [ + "dropNN", + "replaceNN" + ], + "x-enum-description": { + "dropNN": "Drop non-numbers", + "replaceNN": "Replace non-numbers" + } + }, + "replaceWithValue": { + "description": "Only valid when mode is replace", + "type": "number" + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + }, + { + "description": "QueryType = resample", + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions", + "queryType", + "refId" + ], + "properties": { + "datasource": { + "description": "The datasource", + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "description": "The datasource plugin type", + "type": "string", + "pattern": "^__expr__$" + }, + "uid": { + "description": "Datasource UID", + "type": "string" + } + }, + "additionalProperties": false + }, + "downsampler": { + "description": "The reducer", + "type": "string" + }, + "expression": { + "description": "The math expression", + "type": "string" + }, + "hide": { + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)", + "type": "boolean" + }, + "loadedDimensions": { + "type": "object", + "additionalProperties": true, + "x-grafana-type": "data.DataFrame" + }, + "queryType": { + "type": "string", + "pattern": "^resample$" + }, + "refId": { + "description": "RefID is the unique identifier of the query, set by the frontend call.", + "type": "string" + }, + "resultAssertions": { + "description": "Optionally define expected query result behavior", + "type": "object", + "required": [ + "typeVersion" + ], + "properties": { + "maxBytes": { + "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", + "type": "integer" + }, + "maxFrames": { + "description": "Maximum frame count", + "type": "integer" + }, + "type": { + "description": "Type asserts that the frame matches a known type structure.", + "type": "string", + "examples": [ + "timeseries-wide", + "timeseries-long" + ] + }, + "typeVersion": { + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "integer" + } + } + }, + "additionalProperties": false + }, + "timeRange": { + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "description": "From is the start time of the query.", + "type": "string", + "examples": [ + "now-1h" + ] + }, + "to": { + "description": "To is the end time of the query.", + "type": "string", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false + }, + "upsampler": { + "description": "The reducer", + "type": "string" + }, + "window": { + "description": "A time duration string", + "type": "string" + } + }, + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" + } + ], + "$schema": "https://json-schema.org/draft-04/schema" + } + }, + "type": { + "description": "the panel type", + "type": "string" + } + }, + "additionalProperties": true, + "$schema": "https://json-schema.org/draft-04/schema" +} \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.request.example.json b/experimental/resource/schemabuilder/example/query.request.example.json new file mode 100644 index 000000000..871d54dc8 --- /dev/null +++ b/experimental/resource/schemabuilder/example/query.request.example.json @@ -0,0 +1,31 @@ +{ + "from": "now-1h", + "to": "now", + "queries": [ + { + "refId": "A", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A + 10" + }, + { + "refId": "B", + "queryType": "math", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A - $B" + }, + { + "refId": "C", + "queryType": "reduce", + "maxDataPoints": 1000, + "intervalMs": 5, + "expression": "$A", + "reducer": "max", + "settings": { + "mode": "dropNN" + } + } + ] +} \ No newline at end of file From 77f205507eb0f2209c90df2211484036c7e8ed7d Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 20:35:57 -0800 Subject: [PATCH 30/71] now with dataFrame type --- experimental/resource/query.go | 2 +- experimental/resource/query.schema.json | 18 ++++-- experimental/resource/query_parser.go | 24 +++++--- experimental/resource/query_test.go | 56 ------------------- experimental/resource/schemabuilder/enums.go | 15 ++--- .../resource/schemabuilder/enums_test.go | 31 +++++++--- .../resource/schemabuilder/example/query.go | 5 +- .../example/query.panel.schema.json | 54 ++++++++++++++---- .../example/query.request.schema.json | 54 ++++++++++++++---- .../schemabuilder/example/query_test.go | 10 ++-- .../resource/schemabuilder/reflector.go | 42 ++++++++++---- 11 files changed, 182 insertions(+), 129 deletions(-) delete mode 100644 experimental/resource/query_test.go diff --git a/experimental/resource/query.go b/experimental/resource/query.go index 8761ffb2b..37d5a659a 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -124,7 +124,7 @@ type TimeRange struct { // to fail queries *before* returning them to a client (select * from bigquery!) type ResultAssertions struct { // Type asserts that the frame matches a known type structure. - Type data.FrameType `json:"type,omitempty" jsonschema:"example=timeseries-wide,example=timeseries-long"` + Type data.FrameType `json:"type,omitempty"` // TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane // contract documentation https://grafana.github.io/dataplane/contract/. diff --git a/experimental/resource/query.schema.json b/experimental/resource/query.schema.json index 302b3e8e3..f25eac3d4 100644 --- a/experimental/resource/query.schema.json +++ b/experimental/resource/query.schema.json @@ -9,11 +9,21 @@ "properties": { "type": { "type": "string", - "description": "Type asserts that the frame matches a known type structure.", - "examples": [ + "enum": [ + "", "timeseries-wide", - "timeseries-long" - ] + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "x-enum-description": {} }, "typeVersion": { "items": { diff --git a/experimental/resource/query_parser.go b/experimental/resource/query_parser.go index f97bf2b61..8ef3d9cbd 100644 --- a/experimental/resource/query_parser.go +++ b/experimental/resource/query_parser.go @@ -2,6 +2,7 @@ package resource import ( "encoding/json" + "time" "unsafe" "github.com/grafana/grafana-plugin-sdk-go/data/converters" @@ -14,14 +15,6 @@ func init() { //nolint:gochecknoinits jsoniter.RegisterTypeDecoder("resource.GenericDataQuery", &genericQueryCodec{}) } -// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing -type GenericDataQuery struct { - CommonQueryProperties `json:",inline"` - - // Additional Properties (that live at the root) - additional map[string]any `json:"-"` // note this uses custom JSON marshalling -} - type QueryRequest[Q any] struct { // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. // example: now-1h @@ -38,6 +31,17 @@ type QueryRequest[Q any] struct { Debug bool `json:"debug,omitempty"` } +// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing +type GenericDataQuery struct { + CommonQueryProperties `json:",inline"` + + // Additional Properties (that live at the root) + additional map[string]any `json:"-"` // note this uses custom JSON marshalling +} + +// GenericQueryRequest is a query request that supports any datasource +type GenericQueryRequest = QueryRequest[GenericDataQuery] + // Generic query parser pattern. type TypedQueryParser[Q any] interface { // Get the query parser for a query type @@ -47,6 +51,8 @@ type TypedQueryParser[Q any] interface { common CommonQueryProperties, // An iterator with context for the full node (include common values) iter *jsoniter.Iterator, + // Use this value as "now" + now time.Time, ) (Q, error) } @@ -67,7 +73,7 @@ var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) type GenericQueryParser struct{} // ParseQuery implements TypedQueryParser. -func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator) (GenericDataQuery, error) { +func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator, now time.Time) (GenericDataQuery, error) { q := GenericDataQuery{CommonQueryProperties: common, additional: make(map[string]any)} field, err := iter.ReadObject() for field != "" && err == nil { diff --git a/experimental/resource/query_test.go b/experimental/resource/query_test.go deleted file mode 100644 index d3b9272d3..000000000 --- a/experimental/resource/query_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package resource - -import ( - "encoding/json" - "fmt" - "os" - "testing" - - "github.com/invopop/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCommonQueryProperties(t *testing.T) { - r := new(jsonschema.Reflector) - r.DoNotReference = true - err := r.AddGoComments("github.com/grafana/grafana-plugin-sdk-go/experimental/resource", "./") - require.NoError(t, err) - - query := r.Reflect(&CommonQueryProperties{}) - query.ID = "" - query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi - query.Description = "Query properties shared by all data sources" - - // Write the map of values ignored by the common parser - fmt.Printf("var commonKeys = map[string]bool{\n") - for pair := query.Properties.Oldest(); pair != nil; pair = pair.Next() { - fmt.Printf(" \"%s\": true,\n", pair.Key) - } - fmt.Printf("}\n") - - // // Hide this old property - query.Properties.Delete("datasourceId") - out, err := json.MarshalIndent(query, "", " ") - require.NoError(t, err) - - update := false - outfile := "query.schema.json" - body, err := os.ReadFile(outfile) - if err == nil { - if !assert.JSONEq(t, string(out), string(body)) { - update = true - } - } else { - update = true - } - if update { - err = os.WriteFile(outfile, out, 0600) - require.NoError(t, err, "error writing file") - } - - // Make sure the embedded schema is loadable - schema, err := CommonQueryPropertiesSchema() - require.NoError(t, err) - require.Equal(t, 8, len(schema.Properties)) -} diff --git a/experimental/resource/schemabuilder/enums.go b/experimental/resource/schemabuilder/enums.go index 7d9d3c931..b0097ce87 100644 --- a/experimental/resource/schemabuilder/enums.go +++ b/experimental/resource/schemabuilder/enums.go @@ -28,10 +28,10 @@ type EnumField struct { Values []EnumValue } -func findEnumFields(base, path string) ([]EnumField, error) { +func findEnumFields(base, startpath string) ([]EnumField, error) { fset := token.NewFileSet() dict := make(map[string][]*ast.Package) - err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + err := filepath.Walk(startpath, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } @@ -42,7 +42,7 @@ func findEnumFields(base, path string) ([]EnumField, error) { } for _, v := range d { // paths may have multiple packages, like for tests - k := gopath.Join(base, path) + k := gopath.Join(base, strings.TrimPrefix(path, startpath)) dict[k] = append(dict[k], v) } } @@ -72,8 +72,9 @@ func findEnumFields(base, path string) ([]EnumField, error) { txt = gtxt gtxt = "" } - txt = strings.TrimSpace(dp.Synopsis(txt)) + txt = strings.TrimSpace(txt) if strings.HasSuffix(txt, "+enum") { + txt = dp.Synopsis(txt) fields = append(fields, EnumField{ Package: pkg, Name: typ, @@ -87,7 +88,7 @@ func findEnumFields(base, path string) ([]EnumField, error) { if txt == "" { txt = x.Comment.Text() } - if typ == field.Name { + if typ == field.Name && len(x.Values) > 0 { for _, n := range x.Names { if ast.IsExported(n.String()) { v, ok := x.Values[0].(*ast.BasicLit) @@ -118,7 +119,7 @@ func findEnumFields(base, path string) ([]EnumField, error) { // whitespaceRegex is the regex for consecutive whitespaces. var whitespaceRegex = regexp.MustCompile(`\s+`) -func UpdateEnumDescriptions(s *jsonschema.Schema) { +func updateEnumDescriptions(s *jsonschema.Schema) { if len(s.Enum) > 0 && s.Extras != nil { extra, ok := s.Extras["x-enum-description"] if !ok { @@ -146,6 +147,6 @@ func UpdateEnumDescriptions(s *jsonschema.Schema) { } for pair := s.Properties.Oldest(); pair != nil; pair = pair.Next() { - UpdateEnumDescriptions(pair.Value) + updateEnumDescriptions(pair.Value) } } diff --git a/experimental/resource/schemabuilder/enums_test.go b/experimental/resource/schemabuilder/enums_test.go index 745dd5b59..4f368dc12 100644 --- a/experimental/resource/schemabuilder/enums_test.go +++ b/experimental/resource/schemabuilder/enums_test.go @@ -9,14 +9,29 @@ import ( ) func TestFindEnums(t *testing.T) { - fields, err := findEnumFields( - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder", - "./example") - require.NoError(t, err) + t.Run("data", func(t *testing.T) { + fields, err := findEnumFields( + "github.com/grafana/grafana-plugin-sdk-go/data", + "../../../data") + require.NoError(t, err) - out, err := json.MarshalIndent(fields, "", " ") - require.NoError(t, err) - fmt.Printf("%s", string(out)) + out, err := json.MarshalIndent(fields, "", " ") + require.NoError(t, err) + fmt.Printf("%s", string(out)) - require.Equal(t, 3, len(fields)) + require.Equal(t, 1, len(fields)) + }) + + t.Run("example", func(t *testing.T) { + fields, err := findEnumFields( + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder/example", + "./example") + require.NoError(t, err) + + out, err := json.MarshalIndent(fields, "", " ") + require.NoError(t, err) + fmt.Printf("%s", string(out)) + + require.Equal(t, 3, len(fields)) + }) } diff --git a/experimental/resource/schemabuilder/example/query.go b/experimental/resource/schemabuilder/example/query.go index b1bfc5340..d79d4d061 100644 --- a/experimental/resource/schemabuilder/example/query.go +++ b/experimental/resource/schemabuilder/example/query.go @@ -2,6 +2,7 @@ package example import ( "fmt" + "time" "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" @@ -31,12 +32,10 @@ var _ resource.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) type QueyHandler struct{} -// ReadQuery implements query.TypedQueryHandler. func (*QueyHandler) ParseQuery( - // Properties that have been parsed off the same node common resource.CommonQueryProperties, - // An iterator with context for the full node (include common values) iter *jsoniter.Iterator, + now time.Time, ) (ExpressionQuery, error) { qt := QueryType(common.QueryType) switch qt { diff --git a/experimental/resource/schemabuilder/example/query.panel.schema.json b/experimental/resource/schemabuilder/example/query.panel.schema.json index bc11a3237..df260136a 100644 --- a/experimental/resource/schemabuilder/example/query.panel.schema.json +++ b/experimental/resource/schemabuilder/example/query.panel.schema.json @@ -74,12 +74,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", "type": "string", - "examples": [ + "enum": [ + "", "timeseries-wide", - "timeseries-long" - ] + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -199,12 +209,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", "type": "string", - "examples": [ + "enum": [ + "", "timeseries-wide", - "timeseries-long" - ] + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -346,12 +366,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", "type": "string", - "examples": [ + "enum": [ + "", "timeseries-wide", - "timeseries-long" - ] + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", diff --git a/experimental/resource/schemabuilder/example/query.request.schema.json b/experimental/resource/schemabuilder/example/query.request.schema.json index d6c753ee3..531972a6d 100644 --- a/experimental/resource/schemabuilder/example/query.request.schema.json +++ b/experimental/resource/schemabuilder/example/query.request.schema.json @@ -92,12 +92,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", "type": "string", - "examples": [ + "enum": [ + "", "timeseries-wide", - "timeseries-long" - ] + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -225,12 +235,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", "type": "string", - "examples": [ + "enum": [ + "", "timeseries-wide", - "timeseries-long" - ] + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -380,12 +400,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", "type": "string", - "examples": [ + "enum": [ + "", "timeseries-wide", - "timeseries-long" - ] + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", diff --git a/experimental/resource/schemabuilder/example/query_test.go b/experimental/resource/schemabuilder/example/query_test.go index c63299a11..710e7b3c8 100644 --- a/experimental/resource/schemabuilder/example/query_test.go +++ b/experimental/resource/schemabuilder/example/query_test.go @@ -11,11 +11,11 @@ import ( func TestQueryTypeDefinitions(t *testing.T) { builder, err := schemabuilder.NewSchemaBuilder(schemabuilder.BuilderOptions{ - PluginID: []string{"__expr__"}, - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder/example", - CodePath: "./", - // We need to identify the enum fields explicitly :( - // *AND* have the +enum common for this to work + PluginID: []string{"__expr__"}, + ScanCode: []schemabuilder.CodePaths{{ + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder/example", + CodePath: "./", + }}, Enums: []reflect.Type{ reflect.TypeOf(ReducerSum), // pick an example value (not the root) reflect.TypeOf(ReduceModeDrop), // pick an example value (not the root) diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index 79d85665c..261d2ddf0 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -29,15 +29,20 @@ type Builder struct { setting []resource.SettingsDefinition } -type BuilderOptions struct { - // The plugin type ID used in the DataSourceRef type property - PluginID []string - +type CodePaths struct { // ex "github.com/grafana/github-datasource/pkg/models" BasePackage string // ex "./" CodePath string +} + +type BuilderOptions struct { + // The plugin type ID used in the DataSourceRef type property + PluginID []string + + // Scan comments and enumerations + ScanCode []CodePaths // explicitly define the enumeration fields Enums []reflect.Type @@ -72,8 +77,10 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { r := new(jsonschema.Reflector) r.DoNotReference = true - if err := r.AddGoComments(opts.BasePackage, opts.CodePath); err != nil { - return nil, err + for _, scan := range opts.ScanCode { + if err := r.AddGoComments(scan.BasePackage, scan.CodePath); err != nil { + return nil, err + } } customMapper := map[reflect.Type]*jsonschema.Schema{ reflect.TypeOf(data.Frame{}): { @@ -89,10 +96,15 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { } if len(opts.Enums) > 0 { - fields, err := findEnumFields(opts.BasePackage, opts.CodePath) - if err != nil { - return nil, err + fields := []EnumField{} + for _, scan := range opts.ScanCode { + enums, err := findEnumFields(scan.BasePackage, scan.CodePath) + if err != nil { + return nil, err + } + fields = append(fields, enums...) } + for _, etype := range opts.Enums { for _, f := range fields { if f.Name == etype.Name() && f.Package == etype.PkgPath() { @@ -127,8 +139,7 @@ func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { if schema == nil { return fmt.Errorf("missing schema") } - - UpdateEnumDescriptions(schema) + updateEnumDescriptions(schema) name := info.Name if name == "" { @@ -175,7 +186,7 @@ func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { return fmt.Errorf("missing schema") } - UpdateEnumDescriptions(schema) + updateEnumDescriptions(schema) // used by kube-openapi schema.Version = draft04 @@ -301,6 +312,13 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) result := validator.Validate(request) if result.HasErrorsOrWarnings() { + for _, err := range result.Errors { + assert.NoError(t, err) + } + for _, err := range result.Warnings { + assert.NoError(t, err, "warning") + } + body, err = json.MarshalIndent(result, "", " ") require.NoError(t, err) fmt.Printf("Validation: %s\n", string(body)) From f8eb48f7147657885be1fdfe249bee7e78cdb9d3 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 20:36:04 -0800 Subject: [PATCH 31/71] now with dataFrame type --- .../resource/schemabuilder/reflector_test.go | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 experimental/resource/schemabuilder/reflector_test.go diff --git a/experimental/resource/schemabuilder/reflector_test.go b/experimental/resource/schemabuilder/reflector_test.go new file mode 100644 index 000000000..bd41d4b55 --- /dev/null +++ b/experimental/resource/schemabuilder/reflector_test.go @@ -0,0 +1,57 @@ +package schemabuilder + +import ( + "fmt" + "os" + "reflect" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + "github.com/stretchr/testify/require" +) + +func TestWriteQuerySchema(t *testing.T) { + builder, err := NewSchemaBuilder(BuilderOptions{ + PluginID: []string{"dummy"}, + ScanCode: []CodePaths{ + { + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/dotdothack", + CodePath: "../", + }, + { + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/data", + CodePath: "../../../data", + }, + }, + Enums: []reflect.Type{ + reflect.TypeOf(data.FrameTypeLogLines), + }, + }) + require.NoError(t, err) + + query := builder.reflector.Reflect(&resource.CommonQueryProperties{}) + updateEnumDescriptions(query) + query.ID = "" + query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi + query.Description = "Query properties shared by all data sources" + + // Write the map of values ignored by the common parser + fmt.Printf("var commonKeys = map[string]bool{\n") + for pair := query.Properties.Oldest(); pair != nil; pair = pair.Next() { + fmt.Printf(" \"%s\": true,\n", pair.Key) + } + fmt.Printf("}\n") + + // // Hide this old property + query.Properties.Delete("datasourceId") + + outfile := "../query.schema.json" + old, _ := os.ReadFile(outfile) + maybeUpdateFile(t, outfile, query, old) + + // Make sure the embedded schema is loadable + schema, err := resource.CommonQueryPropertiesSchema() + require.NoError(t, err) + require.Equal(t, 8, len(schema.Properties)) +} From 8f647bad87baddb59bdbb56a76eb89cbf79b4702 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 20:41:50 -0800 Subject: [PATCH 32/71] lint --- experimental/resource/metaV1.go | 5 +++++ experimental/resource/query.go | 4 +--- experimental/resource/query_parser.go | 2 +- experimental/resource/schemabuilder/example/query.go | 2 +- experimental/resource/settings.go | 4 +--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/experimental/resource/metaV1.go b/experimental/resource/metaV1.go index a519383c5..47fbee164 100644 --- a/experimental/resource/metaV1.go +++ b/experimental/resource/metaV1.go @@ -12,3 +12,8 @@ type ObjectMeta struct { // Timestamp CreationTimestamp string `json:"creationTimestamp,omitempty"` } + +type TypeMeta struct { + Kind string `json:"kind"` // "QueryTypeDefinitionList", + APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", +} diff --git a/experimental/resource/query.go b/experimental/resource/query.go index 37d5a659a..234e0e256 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -19,9 +19,7 @@ type QueryTypeDefinition struct { // For simple data sources, there may be only a single query type, however when multiple types // exist they must be clearly specified with distinct discriminator field+value pairs type QueryTypeDefinitionList struct { - Kind string `json:"kind"` // "QueryTypeDefinitionList", - APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", - + TypeMeta `json:",inline"` ObjectMeta `json:"metadata,omitempty"` Items []QueryTypeDefinition `json:"items"` diff --git a/experimental/resource/query_parser.go b/experimental/resource/query_parser.go index 8ef3d9cbd..9376f025b 100644 --- a/experimental/resource/query_parser.go +++ b/experimental/resource/query_parser.go @@ -73,7 +73,7 @@ var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) type GenericQueryParser struct{} // ParseQuery implements TypedQueryParser. -func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator, now time.Time) (GenericDataQuery, error) { +func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator, _ time.Time) (GenericDataQuery, error) { q := GenericDataQuery{CommonQueryProperties: common, additional: make(map[string]any)} field, err := iter.ReadObject() for field != "" && err == nil { diff --git a/experimental/resource/schemabuilder/example/query.go b/experimental/resource/schemabuilder/example/query.go index d79d4d061..535026c02 100644 --- a/experimental/resource/schemabuilder/example/query.go +++ b/experimental/resource/schemabuilder/example/query.go @@ -35,7 +35,7 @@ type QueyHandler struct{} func (*QueyHandler) ParseQuery( common resource.CommonQueryProperties, iter *jsoniter.Iterator, - now time.Time, + _ time.Time, ) (ExpressionQuery, error) { qt := QueryType(common.QueryType) switch qt { diff --git a/experimental/resource/settings.go b/experimental/resource/settings.go index 4ef621851..8c9c215c0 100644 --- a/experimental/resource/settings.go +++ b/experimental/resource/settings.go @@ -11,9 +11,7 @@ type SettingsDefinition struct { // For simple data sources, there may be only a single query type, however when multiple types // exist they must be clearly specified with distinct discriminator field+value pairs type SettingsDefinitionList struct { - Kind string `json:"kind"` // "SettingsDefinitionList", - APIVersion string `json:"apiVersion"` // "??.common.grafana.app/v0alpha1", - + TypeMeta `json:",inline"` ObjectMeta `json:"metadata,omitempty"` Items []SettingsDefinition `json:"items"` From 9e14771ddbf882dc35a00f63856f02e2761864f0 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 21:29:30 -0800 Subject: [PATCH 33/71] refactor --- experimental/resource/query.go | 345 ++++++++++++++---- experimental/resource/query.schema.json | 4 +- experimental/resource/query_parser.go | 256 ------------- .../resource/schemabuilder/reflector_test.go | 6 +- experimental/resource/schemabuilder/schema.go | 2 +- go.mod | 2 + go.sum | 2 + 7 files changed, 295 insertions(+), 322 deletions(-) diff --git a/experimental/resource/query.go b/experimental/resource/query.go index 234e0e256..d70eedc28 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -2,62 +2,309 @@ package resource import ( "embed" + "encoding/json" "fmt" + "unsafe" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana-plugin-sdk-go/data/converters" + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + j "github.com/json-iterator/go" + openapi "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/validation/spec" ) -// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition -type QueryTypeDefinition struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` +func init() { //nolint:gochecknoinits + jsoniter.RegisterTypeEncoder("resource.GenericDataQuery", &genericQueryCodec{}) + jsoniter.RegisterTypeDecoder("resource.GenericDataQuery", &genericQueryCodec{}) +} + +// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing +type GenericDataQuery struct { + CommonQueryProperties `json:",inline"` + + // Additional Properties (that live at the root) + additional map[string]any `json:"-"` // note this uses custom JSON marshalling +} + +// Produce an API definition that represents map[string]any +func (g GenericDataQuery) OpenAPIDefinition() openapi.OpenAPIDefinition { + s, _ := GenericQuerySchema() + if s == nil { + s = &spec.Schema{} + } + s.SchemaProps.Type = []string{"object"} + s.SchemaProps.AdditionalProperties = &spec.SchemaOrBool{Allows: true} + s.VendorExtensible = spec.VendorExtensible{ + Extensions: map[string]interface{}{ + "x-kubernetes-preserve-unknown-fields": true, + }, + } + return openapi.OpenAPIDefinition{Schema: *s} +} - Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery. +func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { + if g == nil { + return nil + } + out := new(GenericDataQuery) + jj, err := json.Marshal(g) + if err != nil { + _ = json.Unmarshal(jj, out) + } + return out } -// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types -// For simple data sources, there may be only a single query type, however when multiple types -// exist they must be clearly specified with distinct discriminator field+value pairs -type QueryTypeDefinitionList struct { - TypeMeta `json:",inline"` - ObjectMeta `json:"metadata,omitempty"` +func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { + clone := g.DeepCopy() + *out = *clone +} - Items []QueryTypeDefinition `json:"items"` +// Set allows setting values using key/value pairs +func (g *GenericDataQuery) Set(key string, val any) *GenericDataQuery { + switch key { + case "refId": + g.RefID, _ = val.(string) + case "resultAssertions": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.ResultAssertions) + } + case "timeRange": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.TimeRange) + } + case "datasource": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.Datasource) + } + case "datasourceId": + v, err := converters.JSONValueToInt64.Converter(val) + if err != nil { + g.DatasourceID, _ = v.(int64) + } + case "queryType": + g.QueryType, _ = val.(string) + case "maxDataPoints": + v, err := converters.JSONValueToInt64.Converter(val) + if err != nil { + g.MaxDataPoints, _ = v.(int64) + } + case "intervalMs": + v, err := converters.JSONValueToFloat64.Converter(val) + if err != nil { + g.IntervalMS, _ = v.(float64) + } + case "hide": + g.Hide, _ = val.(bool) + default: + if g.additional == nil { + g.additional = make(map[string]any) + } + g.additional[key] = val + } + return g } -type QueryTypeDefinitionSpec struct { - // Multiple schemas can be defined using discriminators - Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` +func (g *GenericDataQuery) Get(key string) (any, bool) { + switch key { + case "refId": + return g.RefID, true + case "resultAssertions": + return g.ResultAssertions, true + case "timeRange": + return g.TimeRange, true + case "datasource": + return g.Datasource, true + case "datasourceId": + return g.DatasourceID, true + case "queryType": + return g.QueryType, true + case "maxDataPoints": + return g.MaxDataPoints, true + case "intervalMs": + return g.IntervalMS, true + case "hide": + return g.Hide, true + } + v, ok := g.additional[key] + return v, ok +} - // Describe whe the query type is for - Description string `json:"description,omitempty"` +func (g *GenericDataQuery) GetString(key string) string { + v, ok := g.Get(key) + if ok { + s, ok := v.(string) + if ok { + return s + } + } + return "" +} - // The query schema represents the properties that can be sent to the API - // In many cases, this may be the same properties that are saved in a dashboard - // In the case where the save model is different, we must also specify a save model - QuerySchema any `json:"querySchema"` +type genericQueryCodec struct{} - // The save model defines properties that can be saved into dashboard or similar - // These values are processed by frontend components and then sent to the api - // When specified, this schema will be used to validate saved objects rather than - // the query schema - SaveModel any `json:"saveModel,omitempty"` +func (codec *genericQueryCodec) IsEmpty(_ unsafe.Pointer) bool { + return false +} - // Examples (include a wrapper) ideally a template! - Examples []QueryExample `json:"examples,omitempty"` +func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { + q := (*GenericDataQuery)(ptr) + writeQuery(q, stream) +} - // Changelog defines the changed from the previous version - // All changes in the same version *must* be backwards compatible - // Only notable changes will be shown here, for the full version history see git! - Changelog []string `json:"changelog,omitempty"` +func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { + q := GenericDataQuery{} + err := readQuery(&q, jsoniter.NewIterator(iter)) + if err != nil { + // keep existing iter error if it exists + if iter.Error == nil { + iter.Error = err + } + return + } + *((*GenericDataQuery)(ptr)) = q } -type QueryExample struct { - // Version identifier or empty if only one exists - Name string `json:"name,omitempty"` +// MarshalJSON writes JSON including the common and custom values +func (g GenericDataQuery) MarshalJSON() ([]byte, error) { + cfg := j.ConfigCompatibleWithStandardLibrary + stream := cfg.BorrowStream(nil) + defer cfg.ReturnStream(stream) - // An example value saved that can be saved in a dashboard - SaveModel any `json:"saveModel,omitempty"` + writeQuery(&g, stream) + return append([]byte(nil), stream.Buffer()...), stream.Error +} + +// UnmarshalJSON reads a query from json byte array +func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { + iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, b) + if err != nil { + return err + } + return readQuery(g, iter) +} + +func writeQuery(g *GenericDataQuery, stream *j.Stream) { + q := g.CommonQueryProperties + stream.WriteObjectStart() + stream.WriteObjectField("refId") + stream.WriteVal(g.RefID) + + if q.ResultAssertions != nil { + stream.WriteMore() + stream.WriteObjectField("resultAssertions") + stream.WriteVal(g.ResultAssertions) + } + + if q.TimeRange != nil { + stream.WriteMore() + stream.WriteObjectField("timeRange") + stream.WriteVal(g.TimeRange) + } + + if q.Datasource != nil { + stream.WriteMore() + stream.WriteObjectField("datasource") + stream.WriteVal(g.Datasource) + } + + if q.DatasourceID > 0 { + stream.WriteMore() + stream.WriteObjectField("datasourceId") + stream.WriteVal(g.DatasourceID) + } + + if q.QueryType != "" { + stream.WriteMore() + stream.WriteObjectField("queryType") + stream.WriteVal(g.QueryType) + } + + if q.MaxDataPoints > 0 { + stream.WriteMore() + stream.WriteObjectField("maxDataPoints") + stream.WriteVal(g.MaxDataPoints) + } + + if q.IntervalMS > 0 { + stream.WriteMore() + stream.WriteObjectField("intervalMs") + stream.WriteVal(g.IntervalMS) + } + + if q.Hide { + stream.WriteMore() + stream.WriteObjectField("hide") + stream.WriteVal(g.Hide) + } + + // The additional properties + if g.additional != nil { + for k, v := range g.additional { + stream.WriteMore() + stream.WriteObjectField(k) + stream.WriteVal(v) + } + } + stream.WriteObjectEnd() +} + +func readQuery(g *GenericDataQuery, iter *jsoniter.Iterator) error { + var err error + var next j.ValueType + field := "" + for field, err = iter.ReadObject(); field != ""; field, err = iter.ReadObject() { + switch field { + case "refId": + g.RefID, err = iter.ReadString() + case "resultAssertions": + err = iter.ReadVal(&g.ResultAssertions) + case "timeRange": + err = iter.ReadVal(&g.TimeRange) + case "datasource": + // Old datasource values may just be a string + next, err = iter.WhatIsNext() + if err == nil { + switch next { + case j.StringValue: + g.Datasource = &DataSourceRef{} + g.Datasource.UID, err = iter.ReadString() + case j.ObjectValue: + err = iter.ReadVal(&g.Datasource) + default: + return fmt.Errorf("expected string or object") + } + } + + case "datasourceId": + g.DatasourceID, err = iter.ReadInt64() + case "queryType": + g.QueryType, err = iter.ReadString() + case "maxDataPoints": + g.MaxDataPoints, err = iter.ReadInt64() + case "intervalMs": + g.IntervalMS, err = iter.ReadFloat64() + case "hide": + g.Hide, err = iter.ReadBool() + default: + v, err := iter.Read() + if err != nil { + return err + } + if g.additional == nil { + g.additional = make(map[string]any) + } + g.additional[field] = v + } + if err != nil { + return err + } + } + return err } type CommonQueryProperties struct { @@ -135,35 +382,11 @@ type ResultAssertions struct { MaxFrames int64 `json:"maxFrames,omitempty"` } -type DiscriminatorFieldValue struct { - // DiscriminatorField is the field used to link behavior to this specific - // query type. It is typically "queryType", but can be another field if necessary - Field string `json:"field"` - - // The discriminator value - Value string `json:"value"` -} - -// using any since this will often be enumerations -func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { - if len(keyvals)%2 != 0 { - panic("values must be even") - } - dis := []DiscriminatorFieldValue{} - for i := 0; i < len(keyvals); i += 2 { - dis = append(dis, DiscriminatorFieldValue{ - Field: fmt.Sprintf("%v", keyvals[i]), - Value: fmt.Sprintf("%v", keyvals[i+1]), - }) - } - return dis -} - //go:embed query.schema.json var f embed.FS // Get the cached feature list (exposed as a k8s resource) -func CommonQueryPropertiesSchema() (*spec.Schema, error) { +func GenericQuerySchema() (*spec.Schema, error) { body, err := f.ReadFile("query.schema.json") if err != nil { return nil, err diff --git a/experimental/resource/query.schema.json b/experimental/resource/query.schema.json index f25eac3d4..8d98fcf41 100644 --- a/experimental/resource/query.schema.json +++ b/experimental/resource/query.schema.json @@ -110,7 +110,7 @@ "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" } }, - "additionalProperties": false, + "additionalProperties": true, "type": "object", - "description": "Query properties shared by all data sources" + "description": "Generic query properties" } \ No newline at end of file diff --git a/experimental/resource/query_parser.go b/experimental/resource/query_parser.go index 9376f025b..dc8a71eeb 100644 --- a/experimental/resource/query_parser.go +++ b/experimental/resource/query_parser.go @@ -1,20 +1,11 @@ package resource import ( - "encoding/json" "time" - "unsafe" - "github.com/grafana/grafana-plugin-sdk-go/data/converters" "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - j "github.com/json-iterator/go" ) -func init() { //nolint:gochecknoinits - jsoniter.RegisterTypeEncoder("resource.GenericDataQuery", &genericQueryCodec{}) - jsoniter.RegisterTypeDecoder("resource.GenericDataQuery", &genericQueryCodec{}) -} - type QueryRequest[Q any] struct { // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. // example: now-1h @@ -31,14 +22,6 @@ type QueryRequest[Q any] struct { Debug bool `json:"debug,omitempty"` } -// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing -type GenericDataQuery struct { - CommonQueryProperties `json:",inline"` - - // Additional Properties (that live at the root) - additional map[string]any `json:"-"` // note this uses custom JSON marshalling -} - // GenericQueryRequest is a query request that supports any datasource type GenericQueryRequest = QueryRequest[GenericDataQuery] @@ -87,242 +70,3 @@ func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsonit } return q, err } - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery. -func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { - if g == nil { - return nil - } - out := new(GenericDataQuery) - jj, err := json.Marshal(g) - if err != nil { - _ = json.Unmarshal(jj, out) - } - return out -} - -func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { - clone := g.DeepCopy() - *out = *clone -} - -// Set allows setting values using key/value pairs -func (g *GenericDataQuery) Set(key string, val any) *GenericDataQuery { - switch key { - case "refId": - g.RefID, _ = val.(string) - case "resultAssertions": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.ResultAssertions) - } - case "timeRange": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.TimeRange) - } - case "datasource": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.Datasource) - } - case "datasourceId": - v, err := converters.JSONValueToInt64.Converter(val) - if err != nil { - g.DatasourceID, _ = v.(int64) - } - case "queryType": - g.QueryType, _ = val.(string) - case "maxDataPoints": - v, err := converters.JSONValueToInt64.Converter(val) - if err != nil { - g.MaxDataPoints, _ = v.(int64) - } - case "intervalMs": - v, err := converters.JSONValueToFloat64.Converter(val) - if err != nil { - g.IntervalMS, _ = v.(float64) - } - case "hide": - g.Hide, _ = val.(bool) - default: - if g.additional == nil { - g.additional = make(map[string]any) - } - g.additional[key] = val - } - return g -} - -func (g *GenericDataQuery) Get(key string) (any, bool) { - switch key { - case "refId": - return g.RefID, true - case "resultAssertions": - return g.ResultAssertions, true - case "timeRange": - return g.TimeRange, true - case "datasource": - return g.Datasource, true - case "datasourceId": - return g.DatasourceID, true - case "queryType": - return g.QueryType, true - case "maxDataPoints": - return g.MaxDataPoints, true - case "intervalMs": - return g.IntervalMS, true - case "hide": - return g.Hide, true - } - v, ok := g.additional[key] - return v, ok -} - -type genericQueryCodec struct{} - -func (codec *genericQueryCodec) IsEmpty(_ unsafe.Pointer) bool { - return false -} - -func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { - q := (*GenericDataQuery)(ptr) - writeQuery(q, stream) -} - -func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { - q := GenericDataQuery{} - err := readQuery(&q, jsoniter.NewIterator(iter)) - if err != nil { - // keep existing iter error if it exists - if iter.Error == nil { - iter.Error = err - } - return - } - *((*GenericDataQuery)(ptr)) = q -} - -// MarshalJSON writes JSON including the common and custom values -func (g GenericDataQuery) MarshalJSON() ([]byte, error) { - cfg := j.ConfigCompatibleWithStandardLibrary - stream := cfg.BorrowStream(nil) - defer cfg.ReturnStream(stream) - - writeQuery(&g, stream) - return append([]byte(nil), stream.Buffer()...), stream.Error -} - -// UnmarshalJSON reads a query from json byte array -func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { - iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, b) - if err != nil { - return err - } - return readQuery(g, iter) -} - -func writeQuery(g *GenericDataQuery, stream *j.Stream) { - q := g.CommonQueryProperties - stream.WriteObjectStart() - stream.WriteObjectField("refId") - stream.WriteVal(g.RefID) - - if q.ResultAssertions != nil { - stream.WriteMore() - stream.WriteObjectField("resultAssertions") - stream.WriteVal(g.ResultAssertions) - } - - if q.TimeRange != nil { - stream.WriteMore() - stream.WriteObjectField("timeRange") - stream.WriteVal(g.TimeRange) - } - - if q.Datasource != nil { - stream.WriteMore() - stream.WriteObjectField("datasource") - stream.WriteVal(g.Datasource) - } - - if q.DatasourceID > 0 { - stream.WriteMore() - stream.WriteObjectField("datasourceId") - stream.WriteVal(g.DatasourceID) - } - - if q.QueryType != "" { - stream.WriteMore() - stream.WriteObjectField("queryType") - stream.WriteVal(g.QueryType) - } - - if q.MaxDataPoints > 0 { - stream.WriteMore() - stream.WriteObjectField("maxDataPoints") - stream.WriteVal(g.MaxDataPoints) - } - - if q.IntervalMS > 0 { - stream.WriteMore() - stream.WriteObjectField("intervalMs") - stream.WriteVal(g.IntervalMS) - } - - if q.Hide { - stream.WriteMore() - stream.WriteObjectField("hide") - stream.WriteVal(g.Hide) - } - - // The additional properties - if g.additional != nil { - for k, v := range g.additional { - stream.WriteMore() - stream.WriteObjectField(k) - stream.WriteVal(v) - } - } - stream.WriteObjectEnd() -} - -func readQuery(g *GenericDataQuery, iter *jsoniter.Iterator) error { - var err error - field := "" - for field, err = iter.ReadObject(); field != ""; field, err = iter.ReadObject() { - switch field { - case "refId": - g.RefID, err = iter.ReadString() - case "resultAssertions": - err = iter.ReadVal(&g.ResultAssertions) - case "timeRange": - err = iter.ReadVal(&g.TimeRange) - case "datasource": - err = iter.ReadVal(&g.Datasource) - case "datasourceId": - g.DatasourceID, err = iter.ReadInt64() - case "queryType": - g.QueryType, err = iter.ReadString() - case "maxDataPoints": - g.MaxDataPoints, err = iter.ReadInt64() - case "intervalMs": - g.IntervalMS, err = iter.ReadFloat64() - case "hide": - g.Hide, err = iter.ReadBool() - default: - v, err := iter.Read() - if err != nil { - return err - } - if g.additional == nil { - g.additional = make(map[string]any) - } - g.additional[field] = v - } - if err != nil { - return err - } - } - return err -} diff --git a/experimental/resource/schemabuilder/reflector_test.go b/experimental/resource/schemabuilder/reflector_test.go index bd41d4b55..6c512dd0b 100644 --- a/experimental/resource/schemabuilder/reflector_test.go +++ b/experimental/resource/schemabuilder/reflector_test.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + "github.com/invopop/jsonschema" "github.com/stretchr/testify/require" ) @@ -34,7 +35,8 @@ func TestWriteQuerySchema(t *testing.T) { updateEnumDescriptions(query) query.ID = "" query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi - query.Description = "Query properties shared by all data sources" + query.Description = "Generic query properties" + query.AdditionalProperties = jsonschema.TrueSchema // Write the map of values ignored by the common parser fmt.Printf("var commonKeys = map[string]bool{\n") @@ -51,7 +53,7 @@ func TestWriteQuerySchema(t *testing.T) { maybeUpdateFile(t, outfile, query, old) // Make sure the embedded schema is loadable - schema, err := resource.CommonQueryPropertiesSchema() + schema, err := resource.GenericQuerySchema() require.NoError(t, err) require.Equal(t, 8, len(schema.Properties)) } diff --git a/experimental/resource/schemabuilder/schema.go b/experimental/resource/schemabuilder/schema.go index 395e03c21..a53b17ecd 100644 --- a/experimental/resource/schemabuilder/schema.go +++ b/experimental/resource/schemabuilder/schema.go @@ -38,7 +38,7 @@ type QuerySchemaOptions struct { // Given definitions for a plugin, return a valid spec func GetQuerySchema(opts QuerySchemaOptions) (*spec.Schema, error) { isRequest := opts.Mode == SchemaTypeQueryPayload || opts.Mode == SchemaTypeQueryRequest - generic, err := resource.CommonQueryPropertiesSchema() + generic, err := resource.GenericQuerySchema() if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 4ab294250..24ba1e973 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/elazarl/goproxy/ext v0.0.0-20220115173737-adb46da277ac // indirect + github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -76,6 +77,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gofuzz v1.1.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 // indirect github.com/invopop/yaml v0.2.0 // indirect diff --git a/go.sum b/go.sum index 48d28e623..ff1f0f31d 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,8 @@ github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027/go.mod h1:Ro8st/El github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/elazarl/goproxy/ext v0.0.0-20220115173737-adb46da277ac h1:9yrT5tmn9Zc0ytWPASlaPwQfQMQYnRf0RSDe1XvHw0Q= github.com/elazarl/goproxy/ext v0.0.0-20220115173737-adb46da277ac/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From 29beda6f83834489c8db3ad0e159cb8dd9004b09 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 21:32:25 -0800 Subject: [PATCH 34/71] refactor --- experimental/resource/query_definition.go | 79 +++++++++++++++++++++++ experimental/resource/query_test.go | 73 +++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 experimental/resource/query_definition.go create mode 100644 experimental/resource/query_test.go diff --git a/experimental/resource/query_definition.go b/experimental/resource/query_definition.go new file mode 100644 index 000000000..e84fddff4 --- /dev/null +++ b/experimental/resource/query_definition.go @@ -0,0 +1,79 @@ +package resource + +import "fmt" + +// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition +type QueryTypeDefinition struct { + ObjectMeta ObjectMeta `json:"metadata,omitempty"` + + Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` +} + +// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types +// For simple data sources, there may be only a single query type, however when multiple types +// exist they must be clearly specified with distinct discriminator field+value pairs +type QueryTypeDefinitionList struct { + TypeMeta `json:",inline"` + ObjectMeta `json:"metadata,omitempty"` + + Items []QueryTypeDefinition `json:"items"` +} + +type QueryTypeDefinitionSpec struct { + // Multiple schemas can be defined using discriminators + Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // The query schema represents the properties that can be sent to the API + // In many cases, this may be the same properties that are saved in a dashboard + // In the case where the save model is different, we must also specify a save model + QuerySchema any `json:"querySchema"` + + // The save model defines properties that can be saved into dashboard or similar + // These values are processed by frontend components and then sent to the api + // When specified, this schema will be used to validate saved objects rather than + // the query schema + SaveModel any `json:"saveModel,omitempty"` + + // Examples (include a wrapper) ideally a template! + Examples []QueryExample `json:"examples,omitempty"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} + +type QueryExample struct { + // Version identifier or empty if only one exists + Name string `json:"name,omitempty"` + + // An example value saved that can be saved in a dashboard + SaveModel any `json:"saveModel,omitempty"` +} + +type DiscriminatorFieldValue struct { + // DiscriminatorField is the field used to link behavior to this specific + // query type. It is typically "queryType", but can be another field if necessary + Field string `json:"field"` + + // The discriminator value + Value string `json:"value"` +} + +// using any since this will often be enumerations +func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { + if len(keyvals)%2 != 0 { + panic("values must be even") + } + dis := []DiscriminatorFieldValue{} + for i := 0; i < len(keyvals); i += 2 { + dis = append(dis, DiscriminatorFieldValue{ + Field: fmt.Sprintf("%v", keyvals[i]), + Value: fmt.Sprintf("%v", keyvals[i+1]), + }) + } + return dis +} diff --git a/experimental/resource/query_test.go b/experimental/resource/query_test.go new file mode 100644 index 000000000..a4345c079 --- /dev/null +++ b/experimental/resource/query_test.go @@ -0,0 +1,73 @@ +package resource + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseQueriesIntoQueryDataRequest(t *testing.T) { + request := []byte(`{ + "queries": [ + { + "refId": "A", + "datasource": { + "type": "grafana-googlesheets-datasource", + "uid": "b1808c48-9fc9-4045-82d7-081781f8a553" + }, + "cacheDurationSeconds": 300, + "spreadsheet": "spreadsheetID", + "datasourceId": 4, + "intervalMs": 30000, + "maxDataPoints": 794 + }, + { + "refId": "Z", + "datasource": "old", + "maxDataPoints": 10, + "timeRange": { + "from": "100", + "to": "200" + } + } + ], + "from": "1692624667389", + "to": "1692646267389" + }`) + + req := &GenericQueryRequest{} + err := json.Unmarshal(request, req) + require.NoError(t, err) + + require.Len(t, req.Queries, 2) + require.Equal(t, "b1808c48-9fc9-4045-82d7-081781f8a553", req.Queries[0].Datasource.UID) + require.Equal(t, "spreadsheetID", req.Queries[0].GetString("spreadsheet")) + + // Write the query (with additional spreadsheetID) to JSON + out, err := json.MarshalIndent(req.Queries[0], "", " ") + require.NoError(t, err) + + // And read it back with standard JSON marshal functions + query := &GenericDataQuery{} + err = json.Unmarshal(out, query) + require.NoError(t, err) + require.Equal(t, "spreadsheetID", query.GetString("spreadsheet")) + + // The second query has an explicit time range, and legacy datasource name + out, err = json.MarshalIndent(req.Queries[1], "", " ") + require.NoError(t, err) + // fmt.Printf("%s\n", string(out)) + require.JSONEq(t, `{ + "datasource": { + "type": "", ` /* NOTE! this implies legacy naming */ +` + "uid": "old" + }, + "maxDataPoints": 10, + "refId": "Z", + "timeRange": { + "from": "100", + "to": "200" + } + }`, string(out)) +} From a09ab1213090a426857eda936bc4ad64c65a9798 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 25 Feb 2024 21:58:37 -0800 Subject: [PATCH 35/71] another constructor --- experimental/resource/query.go | 12 ++++++++++- experimental/resource/query_test.go | 33 +++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/experimental/resource/query.go b/experimental/resource/query.go index d70eedc28..fa69c33cb 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -43,6 +43,16 @@ func (g GenericDataQuery) OpenAPIDefinition() openapi.OpenAPIDefinition { return openapi.OpenAPIDefinition{Schema: *s} } +func NewGenericDataQuery(body map[string]any) GenericDataQuery { + g := &GenericDataQuery{ + additional: make(map[string]any), + } + for k, v := range body { + _ = g.Set(k, v) + } + return *g +} + // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery. func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { if g == nil { @@ -134,7 +144,7 @@ func (g *GenericDataQuery) Get(key string) (any, bool) { return v, ok } -func (g *GenericDataQuery) GetString(key string) string { +func (g *GenericDataQuery) MustString(key string) string { v, ok := g.Get(key) if ok { s, ok := v.(string) diff --git a/experimental/resource/query_test.go b/experimental/resource/query_test.go index a4345c079..f31f15909 100644 --- a/experimental/resource/query_test.go +++ b/experimental/resource/query_test.go @@ -42,7 +42,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { require.Len(t, req.Queries, 2) require.Equal(t, "b1808c48-9fc9-4045-82d7-081781f8a553", req.Queries[0].Datasource.UID) - require.Equal(t, "spreadsheetID", req.Queries[0].GetString("spreadsheet")) + require.Equal(t, "spreadsheetID", req.Queries[0].MustString("spreadsheet")) // Write the query (with additional spreadsheetID) to JSON out, err := json.MarshalIndent(req.Queries[0], "", " ") @@ -52,7 +52,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { query := &GenericDataQuery{} err = json.Unmarshal(out, query) require.NoError(t, err) - require.Equal(t, "spreadsheetID", query.GetString("spreadsheet")) + require.Equal(t, "spreadsheetID", query.MustString("spreadsheet")) // The second query has an explicit time range, and legacy datasource name out, err = json.MarshalIndent(req.Queries[1], "", " ") @@ -71,3 +71,32 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { } }`, string(out)) } + +func TestQueryBuilders(t *testing.T) { + prop := "testkey" + testQ1 := &GenericDataQuery{} + testQ1.Set(prop, "A") + require.Equal(t, "A", testQ1.MustString(prop)) + + testQ1.Set(prop, "B") + require.Equal(t, "B", testQ1.MustString(prop)) + + testQ2 := testQ1 + testQ2.Set(prop, "C") + require.Equal(t, "C", testQ1.MustString(prop)) + require.Equal(t, "C", testQ2.MustString(prop)) + + // Uses the official field when exists + testQ2.Set("queryType", "D") + require.Equal(t, "D", testQ2.QueryType) + require.Equal(t, "D", testQ1.QueryType) + require.Equal(t, "D", testQ2.MustString("queryType")) + + // Map constructor + testQ3 := NewGenericDataQuery(map[string]any{ + "queryType": "D", + "extra": "E", + }) + require.Equal(t, "D", testQ3.QueryType) + require.Equal(t, "E", testQ3.MustString("extra")) +} From 2151c09f0ff8415918cbaa21fff31b9cb2f09ce9 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 28 Feb 2024 16:13:47 -0800 Subject: [PATCH 36/71] update query parser --- experimental/resource/query.go | 73 +++++++--- experimental/resource/query_parser.go | 130 +++++++++++------- experimental/resource/query_test.go | 74 ++++++---- .../resource/schemabuilder/example/math.go | 46 ------- .../resource/schemabuilder/example/query.go | 99 ++++++++----- .../resource/schemabuilder/example/reduce.go | 57 -------- .../schemabuilder/example/resample.go | 28 ---- .../resource/schemabuilder/examples.go | 8 +- .../resource/schemabuilder/reflector_test.go | 8 -- 9 files changed, 250 insertions(+), 273 deletions(-) delete mode 100644 experimental/resource/schemabuilder/example/math.go delete mode 100644 experimental/resource/schemabuilder/example/reduce.go delete mode 100644 experimental/resource/schemabuilder/example/resample.go diff --git a/experimental/resource/query.go b/experimental/resource/query.go index fa69c33cb..78c528835 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -19,6 +19,35 @@ func init() { //nolint:gochecknoinits jsoniter.RegisterTypeDecoder("resource.GenericDataQuery", &genericQueryCodec{}) } +type DataQuery interface { + // The standard query properties + CommonProperties() *CommonQueryProperties + + // For queries that depend on other queries to run first (eg, other refIds) + Dependencies() []string +} + +type QueryRequest[Q DataQuery] struct { + // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. + // example: now-1h + From string `json:"from,omitempty"` + + // To End time in epoch timestamps in milliseconds or relative using Grafana time units. + // example: now + To string `json:"to,omitempty"` + + // Each item has a + Queries []Q `json:"queries"` + + // required: false + Debug bool `json:"debug,omitempty"` +} + +// GenericQueryRequest is a query request that supports any datasource +type GenericQueryRequest = QueryRequest[*GenericDataQuery] + +var _ DataQuery = (*GenericDataQuery)(nil) + // GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing type GenericDataQuery struct { CommonQueryProperties `json:",inline"` @@ -27,6 +56,16 @@ type GenericDataQuery struct { additional map[string]any `json:"-"` // note this uses custom JSON marshalling } +// CommonProperties implements DataQuery. +func (g *GenericDataQuery) CommonProperties() *CommonQueryProperties { + return &g.CommonQueryProperties +} + +// Dependencies implements DataQuery. +func (g *GenericDataQuery) Dependencies() []string { + return nil +} + // Produce an API definition that represents map[string]any func (g GenericDataQuery) OpenAPIDefinition() openapi.OpenAPIDefinition { s, _ := GenericQuerySchema() @@ -35,11 +74,6 @@ func (g GenericDataQuery) OpenAPIDefinition() openapi.OpenAPIDefinition { } s.SchemaProps.Type = []string{"object"} s.SchemaProps.AdditionalProperties = &spec.SchemaOrBool{Allows: true} - s.VendorExtensible = spec.VendorExtensible{ - Extensions: map[string]interface{}{ - "x-kubernetes-preserve-unknown-fields": true, - }, - } return openapi.OpenAPIDefinition{Schema: *s} } @@ -53,7 +87,6 @@ func NewGenericDataQuery(body map[string]any) GenericDataQuery { return *g } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery. func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { if g == nil { return nil @@ -168,7 +201,7 @@ func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { q := GenericDataQuery{} - err := readQuery(&q, jsoniter.NewIterator(iter)) + err := q.readQuery(jsoniter.NewIterator(iter)) if err != nil { // keep existing iter error if it exists if iter.Error == nil { @@ -195,7 +228,7 @@ func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { if err != nil { return err } - return readQuery(g, iter) + return g.readQuery(iter) } func writeQuery(g *GenericDataQuery, stream *j.Stream) { @@ -263,7 +296,20 @@ func writeQuery(g *GenericDataQuery, stream *j.Stream) { stream.WriteObjectEnd() } -func readQuery(g *GenericDataQuery, iter *jsoniter.Iterator) error { +func (g *GenericDataQuery) readQuery(iter *jsoniter.Iterator) error { + return g.CommonQueryProperties.readQuery(iter, func(key string, iter *jsoniter.Iterator) error { + if g.additional == nil { + g.additional = make(map[string]any) + } + v, err := iter.Read() + g.additional[key] = v + return err + }) +} + +func (g *CommonQueryProperties) readQuery(iter *jsoniter.Iterator, + processUnknownKey func(key string, iter *jsoniter.Iterator) error, +) error { var err error var next j.ValueType field := "" @@ -301,14 +347,7 @@ func readQuery(g *GenericDataQuery, iter *jsoniter.Iterator) error { case "hide": g.Hide, err = iter.ReadBool() default: - v, err := iter.Read() - if err != nil { - return err - } - if g.additional == nil { - g.additional = make(map[string]any) - } - g.additional[field] = v + err = processUnknownKey(field, iter) } if err != nil { return err diff --git a/experimental/resource/query_parser.go b/experimental/resource/query_parser.go index dc8a71eeb..298acfaa2 100644 --- a/experimental/resource/query_parser.go +++ b/experimental/resource/query_parser.go @@ -6,67 +6,93 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" ) -type QueryRequest[Q any] struct { - // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. - // example: now-1h - From string `json:"from,omitempty"` +func ParseQueryRequest(iter *jsoniter.Iterator, now time.Time) (*GenericQueryRequest, error) { + return ParseTypedQueryRequest[*GenericDataQuery](&genericQueryReader{}, iter, time.Now()) +} + +type TypedQueryReader[T DataQuery] interface { + // Called before any custom property is found + Start(p *CommonQueryProperties, now time.Time) error + // Called for each non-common property + SetProperty(key string, iter *jsoniter.Iterator) error + // Finished reading the JSON node + Finish() (T, error) +} - // To End time in epoch timestamps in milliseconds or relative using Grafana time units. - // example: now - To string `json:"to,omitempty"` +func ParseTypedQueryRequest[T DataQuery](reader TypedQueryReader[T], iter *jsoniter.Iterator, now time.Time) (*QueryRequest[T], error) { + var err error + var root string + ok := true + dqr := &QueryRequest[T]{} + for root, err = iter.ReadObject(); root != ""; root, err = iter.ReadObject() { + switch root { + case "to": + dqr.To, err = iter.ReadString() + case "from": + dqr.From, err = iter.ReadString() + case "debug": + dqr.Debug, err = iter.ReadBool() + case "queries": + ok, err = iter.ReadArray() + for ok && err == nil { + props := &CommonQueryProperties{} + err = reader.Start(props, now) + if err != nil { + return dqr, err + } + err = props.readQuery(iter, reader.SetProperty) + if err != nil { + return dqr, err + } - // Each item has a - Queries []Q `json:"queries"` + q, err := reader.Finish() + if err != nil { + return dqr, err + } + dqr.Queries = append(dqr.Queries, q) - // required: false - Debug bool `json:"debug,omitempty"` + ok, err = iter.ReadArray() + if err != nil { + return dqr, err + } + } + default: + // ignored? or error + } + if err != nil { + return dqr, err + } + } + return dqr, err } -// GenericQueryRequest is a query request that supports any datasource -type GenericQueryRequest = QueryRequest[GenericDataQuery] +var _ TypedQueryReader[*GenericDataQuery] = (*genericQueryReader)(nil) -// Generic query parser pattern. -type TypedQueryParser[Q any] interface { - // Get the query parser for a query type - // The version is split from the end of the discriminator field - ParseQuery( - // Properties that have been parsed off the same node - common CommonQueryProperties, - // An iterator with context for the full node (include common values) - iter *jsoniter.Iterator, - // Use this value as "now" - now time.Time, - ) (Q, error) +type genericQueryReader struct { + common *CommonQueryProperties + additional map[string]any } -var commonKeys = map[string]bool{ - "refId": true, - "resultAssertions": true, - "timeRange": true, - "datasource": true, - "datasourceId": true, - "queryType": true, - "maxDataPoints": true, - "intervalMs": true, - "hide": true, +// Called before any custom properties are passed +func (g *genericQueryReader) Start(p *CommonQueryProperties, now time.Time) error { + g.additional = make(map[string]any) + g.common = p + return nil } -var _ TypedQueryParser[GenericDataQuery] = (*GenericQueryParser)(nil) - -type GenericQueryParser struct{} - -// ParseQuery implements TypedQueryParser. -func (*GenericQueryParser) ParseQuery(common CommonQueryProperties, iter *jsoniter.Iterator, _ time.Time) (GenericDataQuery, error) { - q := GenericDataQuery{CommonQueryProperties: common, additional: make(map[string]any)} - field, err := iter.ReadObject() - for field != "" && err == nil { - if !commonKeys[field] { - q.additional[field], err = iter.Read() - if err != nil { - return q, err - } - } - field, err = iter.ReadObject() +func (g *genericQueryReader) SetProperty(key string, iter *jsoniter.Iterator) error { + v, err := iter.Read() + if err != nil { + return err } - return q, err + g.additional[key] = v + return nil +} + +// Finished the JSON node, return a query object +func (g *genericQueryReader) Finish() (*GenericDataQuery, error) { + return &GenericDataQuery{ + CommonQueryProperties: *g.common, + additional: g.additional, + }, nil } diff --git a/experimental/resource/query_test.go b/experimental/resource/query_test.go index f31f15909..40e086a08 100644 --- a/experimental/resource/query_test.go +++ b/experimental/resource/query_test.go @@ -3,7 +3,9 @@ package resource import ( "encoding/json" "testing" + "time" + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" "github.com/stretchr/testify/require" ) @@ -40,36 +42,54 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { err := json.Unmarshal(request, req) require.NoError(t, err) - require.Len(t, req.Queries, 2) - require.Equal(t, "b1808c48-9fc9-4045-82d7-081781f8a553", req.Queries[0].Datasource.UID) - require.Equal(t, "spreadsheetID", req.Queries[0].MustString("spreadsheet")) + t.Run("verify raw unmarshal", func(t *testing.T) { + require.Len(t, req.Queries, 2) + require.Equal(t, "b1808c48-9fc9-4045-82d7-081781f8a553", req.Queries[0].Datasource.UID) + require.Equal(t, "spreadsheetID", req.Queries[0].MustString("spreadsheet")) - // Write the query (with additional spreadsheetID) to JSON - out, err := json.MarshalIndent(req.Queries[0], "", " ") - require.NoError(t, err) + // Write the query (with additional spreadsheetID) to JSON + out, err := json.MarshalIndent(req.Queries[0], "", " ") + require.NoError(t, err) - // And read it back with standard JSON marshal functions - query := &GenericDataQuery{} - err = json.Unmarshal(out, query) - require.NoError(t, err) - require.Equal(t, "spreadsheetID", query.MustString("spreadsheet")) + // And read it back with standard JSON marshal functions + query := &GenericDataQuery{} + err = json.Unmarshal(out, query) + require.NoError(t, err) + require.Equal(t, "spreadsheetID", query.MustString("spreadsheet")) - // The second query has an explicit time range, and legacy datasource name - out, err = json.MarshalIndent(req.Queries[1], "", " ") - require.NoError(t, err) - // fmt.Printf("%s\n", string(out)) - require.JSONEq(t, `{ - "datasource": { - "type": "", ` /* NOTE! this implies legacy naming */ +` - "uid": "old" - }, - "maxDataPoints": 10, - "refId": "Z", - "timeRange": { - "from": "100", - "to": "200" - } - }`, string(out)) + // The second query has an explicit time range, and legacy datasource name + out, err = json.MarshalIndent(req.Queries[1], "", " ") + require.NoError(t, err) + // fmt.Printf("%s\n", string(out)) + require.JSONEq(t, `{ + "datasource": { + "type": "", ` /* NOTE! this implies legacy naming */ +` + "uid": "old" + }, + "maxDataPoints": 10, + "refId": "Z", + "timeRange": { + "from": "100", + "to": "200" + } + }`, string(out)) + }) + + t.Run("same results from either parser", func(t *testing.T) { + iter, err := jsoniter.ParseBytes(jsoniter.ConfigCompatibleWithStandardLibrary, request) + require.NoError(t, err) + + typed, err := ParseQueryRequest(iter, time.Now()) + require.NoError(t, err) + + out1, err := json.MarshalIndent(req, "", " ") + require.NoError(t, err) + + out2, err := json.MarshalIndent(typed, "", " ") + require.NoError(t, err) + + require.JSONEq(t, string(out1), string(out2)) + }) } func TestQueryBuilders(t *testing.T) { diff --git a/experimental/resource/schemabuilder/example/math.go b/experimental/resource/schemabuilder/example/math.go deleted file mode 100644 index 0ef5d43c6..000000000 --- a/experimental/resource/schemabuilder/example/math.go +++ /dev/null @@ -1,46 +0,0 @@ -package example - -import "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - -var _ ExpressionQuery = (*MathQuery)(nil) - -type MathQuery struct { - // General math expression - Expression string `json:"expression" jsonschema:"minLength=1,example=$A + 1,example=$A/$B"` - - // Parsed from the expression - variables []string `json:"-"` -} - -func (*MathQuery) ExpressionQueryType() QueryType { - return QueryTypeMath -} - -func (q *MathQuery) Variables() []string { - return q.variables -} - -func readMathQuery(iter *jsoniter.Iterator) (*MathQuery, error) { - var q *MathQuery - var err error - fname := "" - for fname, err = iter.ReadObject(); fname != "" && err == nil; fname, err = iter.ReadObject() { - switch fname { - case "expression": - temp, err := iter.ReadString() - if err != nil { - return q, err - } - q = &MathQuery{ - Expression: temp, - } - - default: - _, err = iter.ReadAny() // eat up the unused fields - if err != nil { - return nil, err - } - } - } - return q, nil -} diff --git a/experimental/resource/schemabuilder/example/query.go b/experimental/resource/schemabuilder/example/query.go index 535026c02..a9bbbfd75 100644 --- a/experimental/resource/schemabuilder/example/query.go +++ b/experimental/resource/schemabuilder/example/query.go @@ -1,14 +1,7 @@ package example -import ( - "fmt" - "time" +import "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" -) - -// Supported expression types // +enum type QueryType string @@ -23,32 +16,70 @@ const ( QueryTypeResample QueryType = "resample" ) -type ExpressionQuery interface { - ExpressionQueryType() QueryType - Variables() []string +type MathQuery struct { + // General math expression + Expression string `json:"expression" jsonschema:"minLength=1,example=$A + 1,example=$A/$B"` +} + +type ReduceQuery struct { + // Reference to other query results + Expression string `json:"expression"` + + // The reducer + Reducer ReducerID `json:"reducer"` + + // Reducer Options + Settings ReduceSettings `json:"settings"` } -var _ resource.TypedQueryParser[ExpressionQuery] = (*QueyHandler)(nil) - -type QueyHandler struct{} - -func (*QueyHandler) ParseQuery( - common resource.CommonQueryProperties, - iter *jsoniter.Iterator, - _ time.Time, -) (ExpressionQuery, error) { - qt := QueryType(common.QueryType) - switch qt { - case QueryTypeMath: - return readMathQuery(iter) - - case QueryTypeReduce: - q := &ReduceQuery{} - err := iter.ReadVal(q) - return q, err - - case QueryTypeResample: - return nil, nil - } - return nil, fmt.Errorf("unknown query type") +type ReduceSettings struct { + // Non-number reduce behavior + Mode ReduceMode `json:"mode"` + + // Only valid when mode is replace + ReplaceWithValue *float64 `json:"replaceWithValue,omitempty"` +} + +// The reducer function +// +enum +type ReducerID string + +const ( + // The sum + ReducerSum ReducerID = "sum" + // The mean + ReducerMean ReducerID = "mean" + ReducerMin ReducerID = "min" + ReducerMax ReducerID = "max" + ReducerCount ReducerID = "count" + ReducerLast ReducerID = "last" +) + +// Non-Number behavior mode +// +enum +type ReduceMode string + +const ( + // Drop non-numbers + ReduceModeDrop ReduceMode = "dropNN" + + // Replace non-numbers + ReduceModeReplace ReduceMode = "replaceNN" +) + +// QueryType = resample +type ResampleQuery struct { + // The math expression + Expression string `json:"expression"` + + // A time duration string + Window string `json:"window"` + + // The reducer + Downsampler string `json:"downsampler"` + + // The reducer + Upsampler string `json:"upsampler"` + + LoadedDimensions *data.Frame `json:"loadedDimensions"` } diff --git a/experimental/resource/schemabuilder/example/reduce.go b/experimental/resource/schemabuilder/example/reduce.go deleted file mode 100644 index 3893cc06c..000000000 --- a/experimental/resource/schemabuilder/example/reduce.go +++ /dev/null @@ -1,57 +0,0 @@ -package example - -var _ ExpressionQuery = (*ReduceQuery)(nil) - -type ReduceQuery struct { - // Reference to other query results - Expression string `json:"expression"` - - // The reducer - Reducer ReducerID `json:"reducer"` - - // Reducer Options - Settings ReduceSettings `json:"settings"` -} - -func (*ReduceQuery) ExpressionQueryType() QueryType { - return QueryTypeReduce -} - -func (q *ReduceQuery) Variables() []string { - return []string{q.Expression} -} - -type ReduceSettings struct { - // Non-number reduce behavior - Mode ReduceMode `json:"mode"` - - // Only valid when mode is replace - ReplaceWithValue *float64 `json:"replaceWithValue,omitempty"` -} - -// The reducer function -// +enum -type ReducerID string - -const ( - // The sum - ReducerSum ReducerID = "sum" - // The mean - ReducerMean ReducerID = "mean" - ReducerMin ReducerID = "min" - ReducerMax ReducerID = "max" - ReducerCount ReducerID = "count" - ReducerLast ReducerID = "last" -) - -// Non-Number behavior mode -// +enum -type ReduceMode string - -const ( - // Drop non-numbers - ReduceModeDrop ReduceMode = "dropNN" - - // Replace non-numbers - ReduceModeReplace ReduceMode = "replaceNN" -) diff --git a/experimental/resource/schemabuilder/example/resample.go b/experimental/resource/schemabuilder/example/resample.go deleted file mode 100644 index 77f27b1c6..000000000 --- a/experimental/resource/schemabuilder/example/resample.go +++ /dev/null @@ -1,28 +0,0 @@ -package example - -import "github.com/grafana/grafana-plugin-sdk-go/data" - -// QueryType = resample -type ResampleQuery struct { - // The math expression - Expression string `json:"expression"` - - // A time duration string - Window string `json:"window"` - - // The reducer - Downsampler string `json:"downsampler"` - - // The reducer - Upsampler string `json:"upsampler"` - - LoadedDimensions *data.Frame `json:"loadedDimensions"` -} - -func (*ResampleQuery) ExpressionQueryType() QueryType { - return QueryTypeReduce -} - -func (q *ResampleQuery) Variables() []string { - return []string{q.Expression} -} diff --git a/experimental/resource/schemabuilder/examples.go b/experimental/resource/schemabuilder/examples.go index db857c717..fe7261905 100644 --- a/experimental/resource/schemabuilder/examples.go +++ b/experimental/resource/schemabuilder/examples.go @@ -6,11 +6,11 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" ) -func exampleRequest(defs resource.QueryTypeDefinitionList) (resource.QueryRequest[resource.GenericDataQuery], error) { - rsp := resource.QueryRequest[resource.GenericDataQuery]{ +func exampleRequest(defs resource.QueryTypeDefinitionList) (resource.GenericQueryRequest, error) { + rsp := resource.GenericQueryRequest{ From: "now-1h", To: "now", - Queries: []resource.GenericDataQuery{}, + Queries: []*resource.GenericDataQuery{}, } for _, def := range defs.Items { @@ -33,7 +33,7 @@ func exampleRequest(defs resource.QueryTypeDefinitionList) (resource.QueryReques q.IntervalMS = 5 } - rsp.Queries = append(rsp.Queries, *q) + rsp.Queries = append(rsp.Queries, q) } } } diff --git a/experimental/resource/schemabuilder/reflector_test.go b/experimental/resource/schemabuilder/reflector_test.go index 6c512dd0b..27c88135d 100644 --- a/experimental/resource/schemabuilder/reflector_test.go +++ b/experimental/resource/schemabuilder/reflector_test.go @@ -1,7 +1,6 @@ package schemabuilder import ( - "fmt" "os" "reflect" "testing" @@ -38,13 +37,6 @@ func TestWriteQuerySchema(t *testing.T) { query.Description = "Generic query properties" query.AdditionalProperties = jsonschema.TrueSchema - // Write the map of values ignored by the common parser - fmt.Printf("var commonKeys = map[string]bool{\n") - for pair := query.Properties.Oldest(); pair != nil; pair = pair.Next() { - fmt.Printf(" \"%s\": true,\n", pair.Key) - } - fmt.Printf("}\n") - // // Hide this old property query.Properties.Delete("datasourceId") From 5af1ec4ec0809d833d921dd596c748c555d9ec41 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 28 Feb 2024 16:48:15 -0800 Subject: [PATCH 37/71] update query parser --- experimental/resource/query_parser.go | 6 +++--- experimental/resource/query_test.go | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/experimental/resource/query_parser.go b/experimental/resource/query_parser.go index 298acfaa2..a21c7a656 100644 --- a/experimental/resource/query_parser.go +++ b/experimental/resource/query_parser.go @@ -6,7 +6,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" ) -func ParseQueryRequest(iter *jsoniter.Iterator, now time.Time) (*GenericQueryRequest, error) { +func ParseQueryRequest(iter *jsoniter.Iterator) (*GenericQueryRequest, error) { return ParseTypedQueryRequest[*GenericDataQuery](&genericQueryReader{}, iter, time.Now()) } @@ -22,7 +22,7 @@ type TypedQueryReader[T DataQuery] interface { func ParseTypedQueryRequest[T DataQuery](reader TypedQueryReader[T], iter *jsoniter.Iterator, now time.Time) (*QueryRequest[T], error) { var err error var root string - ok := true + var ok bool dqr := &QueryRequest[T]{} for root, err = iter.ReadObject(); root != ""; root, err = iter.ReadObject() { switch root { @@ -74,7 +74,7 @@ type genericQueryReader struct { } // Called before any custom properties are passed -func (g *genericQueryReader) Start(p *CommonQueryProperties, now time.Time) error { +func (g *genericQueryReader) Start(p *CommonQueryProperties, _ time.Time) error { g.additional = make(map[string]any) g.common = p return nil diff --git a/experimental/resource/query_test.go b/experimental/resource/query_test.go index 40e086a08..c349ee9fb 100644 --- a/experimental/resource/query_test.go +++ b/experimental/resource/query_test.go @@ -3,7 +3,6 @@ package resource import ( "encoding/json" "testing" - "time" "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" "github.com/stretchr/testify/require" @@ -79,7 +78,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { iter, err := jsoniter.ParseBytes(jsoniter.ConfigCompatibleWithStandardLibrary, request) require.NoError(t, err) - typed, err := ParseQueryRequest(iter, time.Now()) + typed, err := ParseQueryRequest(iter) require.NoError(t, err) out1, err := json.MarshalIndent(req, "", " ") From f18d1cbd7895035a8cf0e6d6725f3fd4b8dfef57 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 09:37:21 -0800 Subject: [PATCH 38/71] use real type for JSONSchema object --- experimental/resource/query.go | 2 +- experimental/resource/query_definition.go | 11 +-- .../schemabuilder/example/query.types.json | 90 +++++++++---------- .../resource/schemabuilder/reflector.go | 28 ++++-- experimental/resource/schemabuilder/schema.go | 7 +- experimental/resource/settings.go | 6 +- 6 files changed, 76 insertions(+), 68 deletions(-) diff --git a/experimental/resource/query.go b/experimental/resource/query.go index 78c528835..18907afdc 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -93,7 +93,7 @@ func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { } out := new(GenericDataQuery) jj, err := json.Marshal(g) - if err != nil { + if err == nil { _ = json.Unmarshal(jj, out) } return out diff --git a/experimental/resource/query_definition.go b/experimental/resource/query_definition.go index e84fddff4..e946c0818 100644 --- a/experimental/resource/query_definition.go +++ b/experimental/resource/query_definition.go @@ -29,13 +29,7 @@ type QueryTypeDefinitionSpec struct { // The query schema represents the properties that can be sent to the API // In many cases, this may be the same properties that are saved in a dashboard // In the case where the save model is different, we must also specify a save model - QuerySchema any `json:"querySchema"` - - // The save model defines properties that can be saved into dashboard or similar - // These values are processed by frontend components and then sent to the api - // When specified, this schema will be used to validate saved objects rather than - // the query schema - SaveModel any `json:"saveModel,omitempty"` + Schema JSONSchema `json:"schema"` // Examples (include a wrapper) ideally a template! Examples []QueryExample `json:"examples,omitempty"` @@ -50,6 +44,9 @@ type QueryExample struct { // Version identifier or empty if only one exists Name string `json:"name,omitempty"` + // Optionally explain why the example is interesting + Description string `json:"description,omitempty"` + // An example value saved that can be saved in a dashboard SaveModel any `json:"saveModel,omitempty"` } diff --git a/experimental/resource/schemabuilder/example/query.types.json b/experimental/resource/schemabuilder/example/query.types.json index a8d296764..20354866c 100644 --- a/experimental/resource/schemabuilder/example/query.types.json +++ b/experimental/resource/schemabuilder/example/query.types.json @@ -8,7 +8,7 @@ { "metadata": { "name": "math", - "resourceVersion": "1708817676338", + "resourceVersion": "1709227964325", "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { @@ -18,24 +18,24 @@ "value": "math" } ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", - "additionalProperties": false, + "schema": { + "type": "object", + "required": [ + "expression" + ], "properties": { "expression": { "description": "General math expression", + "type": "string", + "minLength": 1, "examples": [ "$A + 1", "$A/$B" - ], - "minLength": 1, - "type": "string" + ] } }, - "required": [ - "expression" - ], - "type": "object" + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" }, "examples": [ { @@ -56,7 +56,7 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1708876267448", + "resourceVersion": "1709227964325", "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { @@ -66,14 +66,20 @@ "value": "reduce" } ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", + "schema": { + "type": "object", + "required": [ + "expression", + "reducer", + "settings" + ], "properties": { "expression": { - "type": "string", - "description": "Reference to other query results" + "description": "Reference to other query results", + "type": "string" }, "reducer": { + "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", "type": "string", "enum": [ "sum", @@ -83,46 +89,40 @@ "count", "last" ], - "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", "x-enum-description": { "mean": "The mean", "sum": "The sum" } }, "settings": { + "description": "Reducer Options", + "type": "object", + "required": [ + "mode" + ], "properties": { "mode": { + "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", "type": "string", "enum": [ "dropNN", "replaceNN" ], - "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", "x-enum-description": { "dropNN": "Drop non-numbers", "replaceNN": "Replace non-numbers" } }, "replaceWithValue": { - "type": "number", - "description": "Only valid when mode is replace" + "description": "Only valid when mode is replace", + "type": "number" } }, - "additionalProperties": false, - "type": "object", - "required": [ - "mode" - ], - "description": "Reducer Options" + "additionalProperties": false } }, "additionalProperties": false, - "type": "object", - "required": [ - "expression", - "reducer", - "settings" - ] + "$schema": "https://json-schema.org/draft-04/schema" }, "examples": [ { @@ -141,7 +141,7 @@ { "metadata": { "name": "resample", - "resourceVersion": "1708548629808", + "resourceVersion": "1709227964325", "creationTimestamp": "2024-02-21T20:50:29Z" }, "spec": { @@ -151,10 +151,16 @@ "value": "resample" } ], - "querySchema": { - "$schema": "https://json-schema.org/draft-04/schema", - "additionalProperties": false, + "schema": { "description": "QueryType = resample", + "type": "object", + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions" + ], "properties": { "downsampler": { "description": "The reducer", @@ -165,8 +171,8 @@ "type": "string" }, "loadedDimensions": { - "additionalProperties": true, "type": "object", + "additionalProperties": true, "x-grafana-type": "data.DataFrame" }, "upsampler": { @@ -178,14 +184,8 @@ "type": "string" } }, - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions" - ], - "type": "object" + "additionalProperties": false, + "$schema": "https://json-schema.org/draft-04/schema" } } } diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index 261d2ddf0..79db66f46 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -51,6 +51,8 @@ type BuilderOptions struct { type QueryTypeInfo struct { // The management name Name string + // Optional description + Description string // Optional discriminators Discriminators []resource.DiscriminatorFieldValue // Raw GO type used for reflection @@ -159,15 +161,22 @@ func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { schema.Version = draft04 schema.ID = "" schema.Anchor = "" + spec, err := asJSONSchema(schema) + if err != nil { + return err + } b.query = append(b.query, resource.QueryTypeDefinition{ ObjectMeta: resource.ObjectMeta{ Name: name, }, Spec: resource.QueryTypeDefinitionSpec{ + Description: info.Description, Discriminators: info.Discriminators, - QuerySchema: schema, - Examples: info.Examples, + Schema: resource.JSONSchema{ + Spec: spec, + }, + Examples: info.Examples, }, }) } @@ -192,6 +201,10 @@ func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { schema.Version = draft04 schema.ID = "" schema.Anchor = "" + spec, err := asJSONSchema(schema) + if err != nil { + return err + } b.setting = append(b.setting, resource.SettingsDefinition{ ObjectMeta: resource.ObjectMeta{ @@ -199,7 +212,9 @@ func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { }, Spec: resource.SettingsDefinitionSpec{ Discriminators: info.Discriminators, - JSONDataSchema: schema, + JSONDataSchema: resource.JSONSchema{ + Spec: spec, + }, }, }) } @@ -241,14 +256,13 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu defs.Items = append(defs.Items, def) } else { - var o1, o2 interface{} b1, _ := json.Marshal(def.Spec) b2, _ := json.Marshal(found.Spec) - _ = json.Unmarshal(b1, &o1) - _ = json.Unmarshal(b2, &o2) - if !reflect.DeepEqual(o1, o2) { + if !assert.JSONEq(&testing.T{}, string(b1), string(b2)) { found.ObjectMeta.ResourceVersion = rv found.Spec = def.Spec + + fmt.Printf("NEW:%s\n", string(b2)) } delete(byName, def.ObjectMeta.Name) } diff --git a/experimental/resource/schemabuilder/schema.go b/experimental/resource/schemabuilder/schema.go index a53b17ecd..edbf0ef74 100644 --- a/experimental/resource/schemabuilder/schema.go +++ b/experimental/resource/schemabuilder/schema.go @@ -75,12 +75,9 @@ func GetQuerySchema(opts QuerySchemaOptions) (*spec.Schema, error) { // The types for each query type queryTypes := []*spec.Schema{} for _, qt := range opts.QueryTypes { - node, err := asJSONSchema(qt.Spec.QuerySchema) - if err != nil { - return nil, fmt.Errorf("error reading query types schema: %s // %w", qt.ObjectMeta.Name, err) - } + node := qt.Spec.Schema.DeepCopy().Spec if node == nil { - return nil, fmt.Errorf("missing query schema: %s // %v", qt.ObjectMeta.Name, qt) + return nil, fmt.Errorf("missing schema for: %s", qt.ObjectMeta.Name) } // Match all discriminators diff --git a/experimental/resource/settings.go b/experimental/resource/settings.go index 8c9c215c0..68f86a2ce 100644 --- a/experimental/resource/settings.go +++ b/experimental/resource/settings.go @@ -27,11 +27,11 @@ type SettingsDefinitionSpec struct { // The query schema represents the properties that can be sent to the API // In many cases, this may be the same properties that are saved in a dashboard // In the case where the save model is different, we must also specify a save model - JSONDataSchema any `json:"jsonDataSchema"` + JSONDataSchema JSONSchema `json:"jsonDataSchema"` // JSON schema defining the properties needed in secure json - // NOTE these must all be string fields - SecureJSONSchema any `json:"secureJsonSchema"` + // NOTE all properties must be string values! + SecureProperties JSONSchema `json:"secureJsonSchema"` // Changelog defines the changed from the previous version // All changes in the same version *must* be backwards compatible From ff38e8818bd9c9a003c2788d7703c3334834440d Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 09:37:30 -0800 Subject: [PATCH 39/71] use real type for JSONSchema object --- experimental/resource/schema.go | 60 ++++++++++++++++++++++++++++ experimental/resource/schema_test.go | 26 ++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 experimental/resource/schema.go create mode 100644 experimental/resource/schema_test.go diff --git a/experimental/resource/schema.go b/experimental/resource/schema.go new file mode 100644 index 000000000..d68e7fa01 --- /dev/null +++ b/experimental/resource/schema.go @@ -0,0 +1,60 @@ +package resource + +import ( + "encoding/json" + + openapi "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +// The k8s compatible jsonschema version +const draft04 = "https://json-schema.org/draft-04/schema#" + +type JSONSchema struct { + Spec *spec.Schema +} + +func (s JSONSchema) MarshalJSON() ([]byte, error) { + if s.Spec == nil { + return []byte("{}"), nil + } + return s.Spec.MarshalJSON() +} + +func (s *JSONSchema) UnmarshalJSON(data []byte) error { + s.Spec = &spec.Schema{} + return s.Spec.UnmarshalJSON(data) +} + +func (g JSONSchema) OpenAPIDefinition() openapi.OpenAPIDefinition { + return openapi.OpenAPIDefinition{Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: spec.MustCreateRef(draft04), + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + }} +} + +func (g *JSONSchema) DeepCopy() *JSONSchema { + if g == nil { + return nil + } + out := &JSONSchema{} + if g.Spec != nil { + out.Spec = &spec.Schema{} + jj, err := json.Marshal(g.Spec) + if err == nil { + _ = json.Unmarshal(jj, out.Spec) + } + } + return out +} + +func (g *JSONSchema) DeepCopyInto(out *JSONSchema) { + if g.Spec == nil { + out.Spec = nil + return + } + out.Spec = g.DeepCopy().Spec +} diff --git a/experimental/resource/schema_test.go b/experimental/resource/schema_test.go new file mode 100644 index 000000000..e7ae50807 --- /dev/null +++ b/experimental/resource/schema_test.go @@ -0,0 +1,26 @@ +package resource + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +func TestSchemaSupport(t *testing.T) { + val := JSONSchema{ + Spec: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "hello", + }, + }, + } + jj, err := json.Marshal(val) + require.NoError(t, err) + + copy := &JSONSchema{} + err = copy.UnmarshalJSON(jj) + require.NoError(t, err) + require.Equal(t, val.Spec.Description, copy.Spec.Description) +} From 34c635a5bbe3161e87d63ab71d51f03c97775baa Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 10:32:08 -0800 Subject: [PATCH 40/71] use real type for JSONSchema object --- experimental/resource/schema.go | 12 ++- experimental/resource/schema_test.go | 7 +- .../example/query.panel.schema.json | 4 +- .../example/query.request.schema.json | 4 +- .../schemabuilder/example/query.types.json | 84 +++++++++---------- .../resource/schemabuilder/reflector_test.go | 2 +- experimental/resource/schemabuilder/schema.go | 2 +- 7 files changed, 65 insertions(+), 50 deletions(-) diff --git a/experimental/resource/schema.go b/experimental/resource/schema.go index d68e7fa01..3d994850c 100644 --- a/experimental/resource/schema.go +++ b/experimental/resource/schema.go @@ -18,7 +18,17 @@ func (s JSONSchema) MarshalJSON() ([]byte, error) { if s.Spec == nil { return []byte("{}"), nil } - return s.Spec.MarshalJSON() + body, err := s.Spec.MarshalJSON() + if err == nil { + // The internal format puts $schema last! + // this moves $schema first + copy := map[string]any{} + err := json.Unmarshal(body, ©) + if err == nil { + return json.Marshal(copy) + } + } + return body, err } func (s *JSONSchema) UnmarshalJSON(data []byte) error { diff --git a/experimental/resource/schema_test.go b/experimental/resource/schema_test.go index e7ae50807..ea0b76ac4 100644 --- a/experimental/resource/schema_test.go +++ b/experimental/resource/schema_test.go @@ -2,6 +2,7 @@ package resource import ( "encoding/json" + "fmt" "testing" "github.com/stretchr/testify/require" @@ -13,12 +14,16 @@ func TestSchemaSupport(t *testing.T) { Spec: &spec.Schema{ SchemaProps: spec.SchemaProps{ Description: "hello", + Schema: draft04, + ID: "something", }, }, } - jj, err := json.Marshal(val) + jj, err := json.MarshalIndent(val, "", "") require.NoError(t, err) + fmt.Printf("%s\n", string(jj)) + copy := &JSONSchema{} err = copy.UnmarshalJSON(jj) require.NoError(t, err) diff --git a/experimental/resource/schemabuilder/example/query.panel.schema.json b/experimental/resource/schemabuilder/example/query.panel.schema.json index df260136a..a9c19e5b0 100644 --- a/experimental/resource/schemabuilder/example/query.panel.schema.json +++ b/experimental/resource/schemabuilder/example/query.panel.schema.json @@ -433,7 +433,7 @@ "$schema": "https://json-schema.org/draft-04/schema" } ], - "$schema": "https://json-schema.org/draft-04/schema" + "$schema": "https://json-schema.org/draft-04/schema#" } }, "type": { @@ -442,5 +442,5 @@ } }, "additionalProperties": true, - "$schema": "https://json-schema.org/draft-04/schema" + "$schema": "https://json-schema.org/draft-04/schema#" } \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.request.schema.json b/experimental/resource/schemabuilder/example/query.request.schema.json index 531972a6d..fc9d16478 100644 --- a/experimental/resource/schemabuilder/example/query.request.schema.json +++ b/experimental/resource/schemabuilder/example/query.request.schema.json @@ -467,7 +467,7 @@ "$schema": "https://json-schema.org/draft-04/schema" } ], - "$schema": "https://json-schema.org/draft-04/schema" + "$schema": "https://json-schema.org/draft-04/schema#" } }, "to": { @@ -476,5 +476,5 @@ } }, "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" + "$schema": "https://json-schema.org/draft-04/schema#" } \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.types.json b/experimental/resource/schemabuilder/example/query.types.json index 20354866c..24d5e03d4 100644 --- a/experimental/resource/schemabuilder/example/query.types.json +++ b/experimental/resource/schemabuilder/example/query.types.json @@ -2,14 +2,14 @@ "kind": "QueryTypeDefinitionList", "apiVersion": "query.grafana.app/v0alpha1", "metadata": { - "resourceVersion": "1708548629808" + "resourceVersion": "1709230013217" }, "items": [ { "metadata": { "name": "math", - "resourceVersion": "1709227964325", - "creationTimestamp": "2024-02-21T20:50:29Z" + "resourceVersion": "1709230013217", + "creationTimestamp": "2024-02-29T18:06:53Z" }, "spec": { "discriminators": [ @@ -19,23 +19,23 @@ } ], "schema": { - "type": "object", - "required": [ - "expression" - ], + "$schema": "https://json-schema.org/draft-04/schema", + "additionalProperties": false, "properties": { "expression": { "description": "General math expression", - "type": "string", - "minLength": 1, "examples": [ "$A + 1", "$A/$B" - ] + ], + "minLength": 1, + "type": "string" } }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" + "required": [ + "expression" + ], + "type": "object" }, "examples": [ { @@ -56,8 +56,8 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1709227964325", - "creationTimestamp": "2024-02-21T20:50:29Z" + "resourceVersion": "1709230013217", + "creationTimestamp": "2024-02-29T18:06:53Z" }, "spec": { "discriminators": [ @@ -67,12 +67,8 @@ } ], "schema": { - "type": "object", - "required": [ - "expression", - "reducer", - "settings" - ], + "$schema": "https://json-schema.org/draft-04/schema", + "additionalProperties": false, "properties": { "expression": { "description": "Reference to other query results", @@ -80,7 +76,6 @@ }, "reducer": { "description": "The reducer\n\n\nPossible enum values:\n - `\"sum\"` The sum\n - `\"mean\"` The mean\n - `\"min\"` \n - `\"max\"` \n - `\"count\"` \n - `\"last\"` ", - "type": "string", "enum": [ "sum", "mean", @@ -89,25 +84,23 @@ "count", "last" ], + "type": "string", "x-enum-description": { "mean": "The mean", "sum": "The sum" } }, "settings": { + "additionalProperties": false, "description": "Reducer Options", - "type": "object", - "required": [ - "mode" - ], "properties": { "mode": { "description": "Non-number reduce behavior\n\n\nPossible enum values:\n - `\"dropNN\"` Drop non-numbers\n - `\"replaceNN\"` Replace non-numbers", - "type": "string", "enum": [ "dropNN", "replaceNN" ], + "type": "string", "x-enum-description": { "dropNN": "Drop non-numbers", "replaceNN": "Replace non-numbers" @@ -118,11 +111,18 @@ "type": "number" } }, - "additionalProperties": false + "required": [ + "mode" + ], + "type": "object" } }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" + "required": [ + "expression", + "reducer", + "settings" + ], + "type": "object" }, "examples": [ { @@ -141,8 +141,8 @@ { "metadata": { "name": "resample", - "resourceVersion": "1709227964325", - "creationTimestamp": "2024-02-21T20:50:29Z" + "resourceVersion": "1709230013217", + "creationTimestamp": "2024-02-29T18:06:53Z" }, "spec": { "discriminators": [ @@ -152,15 +152,9 @@ } ], "schema": { + "$schema": "https://json-schema.org/draft-04/schema", + "additionalProperties": false, "description": "QueryType = resample", - "type": "object", - "required": [ - "expression", - "window", - "downsampler", - "upsampler", - "loadedDimensions" - ], "properties": { "downsampler": { "description": "The reducer", @@ -171,8 +165,8 @@ "type": "string" }, "loadedDimensions": { - "type": "object", "additionalProperties": true, + "type": "object", "x-grafana-type": "data.DataFrame" }, "upsampler": { @@ -184,8 +178,14 @@ "type": "string" } }, - "additionalProperties": false, - "$schema": "https://json-schema.org/draft-04/schema" + "required": [ + "expression", + "window", + "downsampler", + "upsampler", + "loadedDimensions" + ], + "type": "object" } } } diff --git a/experimental/resource/schemabuilder/reflector_test.go b/experimental/resource/schemabuilder/reflector_test.go index 27c88135d..172d5492e 100644 --- a/experimental/resource/schemabuilder/reflector_test.go +++ b/experimental/resource/schemabuilder/reflector_test.go @@ -33,7 +33,7 @@ func TestWriteQuerySchema(t *testing.T) { query := builder.reflector.Reflect(&resource.CommonQueryProperties{}) updateEnumDescriptions(query) query.ID = "" - query.Version = "https://json-schema.org/draft-04/schema" // used by kube-openapi + query.Version = draft04 // used by kube-openapi query.Description = "Generic query properties" query.AdditionalProperties = jsonschema.TrueSchema diff --git a/experimental/resource/schemabuilder/schema.go b/experimental/resource/schemabuilder/schema.go index edbf0ef74..e13194ab3 100644 --- a/experimental/resource/schemabuilder/schema.go +++ b/experimental/resource/schemabuilder/schema.go @@ -9,7 +9,7 @@ import ( ) // The k8s compatible jsonschema version -const draft04 = "https://json-schema.org/draft-04/schema" +const draft04 = "https://json-schema.org/draft-04/schema#" // Supported expression types // +enum From 0847e1d0c8400ccf405a5073953dbe271db371c6 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 11:44:26 -0800 Subject: [PATCH 41/71] add spec definition --- .../resource/query.definition.schema.json | 69 +++++++++++++++++++ experimental/resource/query.go | 2 +- experimental/resource/query.schema.json | 2 +- experimental/resource/query_definition.go | 35 +++++++++- .../resource/schemabuilder/reflector.go | 4 ++ .../resource/schemabuilder/reflector_test.go | 12 ++++ 6 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 experimental/resource/query.definition.schema.json diff --git a/experimental/resource/query.definition.schema.json b/experimental/resource/query.definition.schema.json new file mode 100644 index 000000000..e23f11d3c --- /dev/null +++ b/experimental/resource/query.definition.schema.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema#", + "properties": { + "discriminators": { + "items": { + "properties": { + "field": { + "type": "string", + "description": "DiscriminatorField is the field used to link behavior to this specific\nquery type. It is typically \"queryType\", but can be another field if necessary" + }, + "value": { + "type": "string", + "description": "The discriminator value" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "field", + "value" + ] + }, + "type": "array", + "description": "Multiple schemas can be defined using discriminators" + }, + "description": { + "type": "string", + "description": "Describe whe the query type is for" + }, + "schema": { + "$ref": "https://json-schema.org/draft-04/schema#", + "type": "object", + "description": "The query schema represents the properties that can be sent to the API\nIn many cases, this may be the same properties that are saved in a dashboard\nIn the case where the save model is different, we must also specify a save model" + }, + "examples": { + "items": { + "properties": { + "name": { + "type": "string", + "description": "Version identifier or empty if only one exists" + }, + "description": { + "type": "string", + "description": "Optionally explain why the example is interesting" + }, + "saveModel": { + "description": "An example value saved that can be saved in a dashboard" + } + }, + "additionalProperties": false, + "type": "object" + }, + "type": "array", + "description": "Examples (include a wrapper) ideally a template!" + }, + "changelog": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Changelog defines the changed from the previous version\nAll changes in the same version *must* be backwards compatible\nOnly notable changes will be shown here, for the full version history see git!" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "schema" + ] +} \ No newline at end of file diff --git a/experimental/resource/query.go b/experimental/resource/query.go index 18907afdc..a9e1f59a1 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -431,7 +431,7 @@ type ResultAssertions struct { MaxFrames int64 `json:"maxFrames,omitempty"` } -//go:embed query.schema.json +//go:embed query.schema.json query.definition.schema.json var f embed.FS // Get the cached feature list (exposed as a k8s resource) diff --git a/experimental/resource/query.schema.json b/experimental/resource/query.schema.json index 8d98fcf41..7c21a8053 100644 --- a/experimental/resource/query.schema.json +++ b/experimental/resource/query.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-04/schema", + "$schema": "https://json-schema.org/draft-04/schema#", "properties": { "refId": { "type": "string", diff --git a/experimental/resource/query_definition.go b/experimental/resource/query_definition.go index e946c0818..0dbd95fcc 100644 --- a/experimental/resource/query_definition.go +++ b/experimental/resource/query_definition.go @@ -1,6 +1,12 @@ package resource -import "fmt" +import ( + "encoding/json" + "fmt" + + openapi "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/validation/spec" +) // QueryTypeDefinition is a kubernetes shaped object that represents a single query definition type QueryTypeDefinition struct { @@ -40,6 +46,33 @@ type QueryTypeDefinitionSpec struct { Changelog []string `json:"changelog,omitempty"` } +// Produce an API definition that represents map[string]any +func (g QueryTypeDefinitionSpec) OpenAPIDefinition() openapi.OpenAPIDefinition { + s := &spec.Schema{} + body, err := f.ReadFile("query.definition.schema.json") + if err == nil { + _ = s.UnmarshalJSON(body) + } + return openapi.OpenAPIDefinition{Schema: *s} +} + +func (g *QueryTypeDefinitionSpec) DeepCopy() *QueryTypeDefinitionSpec { + if g == nil { + return nil + } + out := new(QueryTypeDefinitionSpec) + jj, err := json.Marshal(g) + if err == nil { + _ = json.Unmarshal(jj, out) + } + return out +} + +func (g *QueryTypeDefinitionSpec) DeepCopyInto(out *QueryTypeDefinitionSpec) { + clone := g.DeepCopy() + *out = *clone +} + type QueryExample struct { // Version identifier or empty if only one exists Name string `json:"name,omitempty"` diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index 79db66f46..c7b251372 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -92,6 +92,10 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { }, AdditionalProperties: jsonschema.TrueSchema, }, + reflect.TypeOf(resource.JSONSchema{}): { + Type: "object", + Ref: draft04, + }, } r.Mapper = func(t reflect.Type) *jsonschema.Schema { return customMapper[t] diff --git a/experimental/resource/schemabuilder/reflector_test.go b/experimental/resource/schemabuilder/reflector_test.go index 172d5492e..f64b4c2af 100644 --- a/experimental/resource/schemabuilder/reflector_test.go +++ b/experimental/resource/schemabuilder/reflector_test.go @@ -48,4 +48,16 @@ func TestWriteQuerySchema(t *testing.T) { schema, err := resource.GenericQuerySchema() require.NoError(t, err) require.Equal(t, 8, len(schema.Properties)) + + // Add schema for query type definition + query = builder.reflector.Reflect(&resource.QueryTypeDefinitionSpec{}) + updateEnumDescriptions(query) + query.ID = "" + query.Version = draft04 // used by kube-openapi + outfile = "../query.definition.schema.json" + old, _ = os.ReadFile(outfile) + maybeUpdateFile(t, outfile, query, old) + + def := resource.QueryTypeDefinitionSpec{}.OpenAPIDefinition() + require.Equal(t, query.Properties.Len(), len(def.Schema.Properties)) } From 848210935f0886691b3b5e6f89ccbf4dc9ec3515 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 12:42:02 -0800 Subject: [PATCH 42/71] more specs --- experimental/resource/query.go | 28 ------------------- experimental/resource/query_definition.go | 13 --------- .../resource/schemabuilder/reflector_test.go | 2 +- 3 files changed, 1 insertion(+), 42 deletions(-) diff --git a/experimental/resource/query.go b/experimental/resource/query.go index a9e1f59a1..5edea77bc 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -1,7 +1,6 @@ package resource import ( - "embed" "encoding/json" "fmt" "unsafe" @@ -10,8 +9,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data/converters" "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" j "github.com/json-iterator/go" - openapi "k8s.io/kube-openapi/pkg/common" - "k8s.io/kube-openapi/pkg/validation/spec" ) func init() { //nolint:gochecknoinits @@ -66,17 +63,6 @@ func (g *GenericDataQuery) Dependencies() []string { return nil } -// Produce an API definition that represents map[string]any -func (g GenericDataQuery) OpenAPIDefinition() openapi.OpenAPIDefinition { - s, _ := GenericQuerySchema() - if s == nil { - s = &spec.Schema{} - } - s.SchemaProps.Type = []string{"object"} - s.SchemaProps.AdditionalProperties = &spec.SchemaOrBool{Allows: true} - return openapi.OpenAPIDefinition{Schema: *s} -} - func NewGenericDataQuery(body map[string]any) GenericDataQuery { g := &GenericDataQuery{ additional: make(map[string]any), @@ -430,17 +416,3 @@ type ResultAssertions struct { // Maximum frame count MaxFrames int64 `json:"maxFrames,omitempty"` } - -//go:embed query.schema.json query.definition.schema.json -var f embed.FS - -// Get the cached feature list (exposed as a k8s resource) -func GenericQuerySchema() (*spec.Schema, error) { - body, err := f.ReadFile("query.schema.json") - if err != nil { - return nil, err - } - s := &spec.Schema{} - err = s.UnmarshalJSON(body) - return s, err -} diff --git a/experimental/resource/query_definition.go b/experimental/resource/query_definition.go index 0dbd95fcc..132501456 100644 --- a/experimental/resource/query_definition.go +++ b/experimental/resource/query_definition.go @@ -3,9 +3,6 @@ package resource import ( "encoding/json" "fmt" - - openapi "k8s.io/kube-openapi/pkg/common" - "k8s.io/kube-openapi/pkg/validation/spec" ) // QueryTypeDefinition is a kubernetes shaped object that represents a single query definition @@ -46,16 +43,6 @@ type QueryTypeDefinitionSpec struct { Changelog []string `json:"changelog,omitempty"` } -// Produce an API definition that represents map[string]any -func (g QueryTypeDefinitionSpec) OpenAPIDefinition() openapi.OpenAPIDefinition { - s := &spec.Schema{} - body, err := f.ReadFile("query.definition.schema.json") - if err == nil { - _ = s.UnmarshalJSON(body) - } - return openapi.OpenAPIDefinition{Schema: *s} -} - func (g *QueryTypeDefinitionSpec) DeepCopy() *QueryTypeDefinitionSpec { if g == nil { return nil diff --git a/experimental/resource/schemabuilder/reflector_test.go b/experimental/resource/schemabuilder/reflector_test.go index f64b4c2af..e7bb74361 100644 --- a/experimental/resource/schemabuilder/reflector_test.go +++ b/experimental/resource/schemabuilder/reflector_test.go @@ -58,6 +58,6 @@ func TestWriteQuerySchema(t *testing.T) { old, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) - def := resource.QueryTypeDefinitionSpec{}.OpenAPIDefinition() + def := resource.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/experimental/resource.QueryTypeDefinitionSpec"] require.Equal(t, query.Properties.Len(), len(def.Schema.Properties)) } From 8dbdba07e08aebc8207a65981607aa6527105edf Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 12:42:09 -0800 Subject: [PATCH 43/71] more specs --- experimental/resource/openapi.go | 92 +++++++++++++++++++ experimental/resource/openapi_test.go | 40 ++++++++ .../testdata/sample_query_results.json | 59 ++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 experimental/resource/openapi.go create mode 100644 experimental/resource/openapi_test.go create mode 100644 experimental/resource/testdata/sample_query_results.json diff --git a/experimental/resource/openapi.go b/experimental/resource/openapi.go new file mode 100644 index 000000000..3369fb270 --- /dev/null +++ b/experimental/resource/openapi.go @@ -0,0 +1,92 @@ +package resource + +import ( + "embed" + + common "k8s.io/kube-openapi/pkg/common" + openapi "k8s.io/kube-openapi/pkg/common" + spec "k8s.io/kube-openapi/pkg/validation/spec" +) + +//go:embed query.schema.json query.definition.schema.json +var f embed.FS + +func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + return map[string]common.OpenAPIDefinition{ + "github.com/grafana/grafana-plugin-sdk-go/backend.QueryDataResponse": schema_backend_query_data_response(ref), + "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schema_data_frame(ref), + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.GenericDataQuery": schema_GenericQuery(ref), + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.QueryTypeDefinitionSpec": schema_QueryTypeDefinitionSpec(ref), + } +} + +func schema_backend_query_data_response(_ common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "results keyed by refId", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "results": *spec.MapProperty(&spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "any object for now", + Type: []string{"object"}, + Properties: map[string]spec.Schema{}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + }), + }, + AdditionalProperties: &spec.SchemaOrBool{Allows: false}, + }, + }, + } +} + +func schema_data_frame(_ common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "any object for now", + Type: []string{"object"}, + Properties: map[string]spec.Schema{}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + }, + } +} + +func schema_QueryTypeDefinitionSpec(_ common.ReferenceCallback) common.OpenAPIDefinition { + s, _ := loadSchema("query.definition.schema.json") + if s == nil { + s = &spec.Schema{} + } + return common.OpenAPIDefinition{ + Schema: *s, + } +} + +func schema_GenericQuery(_ common.ReferenceCallback) common.OpenAPIDefinition { + s, _ := GenericQuerySchema() + if s == nil { + s = &spec.Schema{} + } + s.SchemaProps.Type = []string{"object"} + s.SchemaProps.AdditionalProperties = &spec.SchemaOrBool{Allows: true} + return openapi.OpenAPIDefinition{Schema: *s} +} + +// Get the cached feature list (exposed as a k8s resource) +func GenericQuerySchema() (*spec.Schema, error) { + return loadSchema("query.schema.json") +} + +// Get the cached feature list (exposed as a k8s resource) +func loadSchema(path string) (*spec.Schema, error) { + body, err := f.ReadFile(path) + if err != nil { + return nil, err + } + s := &spec.Schema{} + err = s.UnmarshalJSON(body) + return s, err +} diff --git a/experimental/resource/openapi_test.go b/experimental/resource/openapi_test.go new file mode 100644 index 000000000..74def6589 --- /dev/null +++ b/experimental/resource/openapi_test.go @@ -0,0 +1,40 @@ +package resource + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/kube-openapi/pkg/validation/spec" + "k8s.io/kube-openapi/pkg/validation/strfmt" + "k8s.io/kube-openapi/pkg/validation/validate" +) + +func TestOpenAPI(t *testing.T) { + //nolint:gocritic + defs := GetOpenAPIDefinitions(func(path string) spec.Ref { // (unlambda: replace ¯\_(ツ)_/¯) + return spec.MustCreateRef(path) // placeholder for tests + }) + + def, ok := defs["github.com/grafana/grafana-plugin-sdk-go/backend.QueryDataResponse"] + require.True(t, ok) + require.Empty(t, def.Dependencies) // not yet supported! + + validator := validate.NewSchemaValidator(&def.Schema, nil, "data", strfmt.Default) + + body, err := os.ReadFile("./testdata/sample_query_results.json") + require.NoError(t, err) + unstructured := make(map[string]any) + err = json.Unmarshal(body, &unstructured) + require.NoError(t, err) + + result := validator.Validate(unstructured) + for _, err := range result.Errors { + assert.NoError(t, err, "validation error") + } + for _, err := range result.Warnings { + assert.NoError(t, err, "validation warning") + } +} diff --git a/experimental/resource/testdata/sample_query_results.json b/experimental/resource/testdata/sample_query_results.json new file mode 100644 index 000000000..30de119d4 --- /dev/null +++ b/experimental/resource/testdata/sample_query_results.json @@ -0,0 +1,59 @@ +{ + "results": { + "A": { + "status": 200, + "frames": [ + { + "schema": { + "refId": "A", + "meta": { + "typeVersion": [0, 0], + "custom": { + "customStat": 10 + } + }, + "fields": [ + { + "name": "time", + "type": "time", + "typeInfo": { + "frame": "time.Time", + "nullable": true + }, + "config": { + "interval": 1800000 + } + }, + { + "name": "A-series", + "type": "number", + "typeInfo": { + "frame": "float64", + "nullable": true + }, + "labels": {} + } + ] + }, + "data": { + "values": [ + [ + 1708955198367, 1708956998367, 1708958798367, 1708960598367, 1708962398367, 1708964198367, 1708965998367, + 1708967798367, 1708969598367, 1708971398367, 1708973198367, 1708974998367 + ], + [ + 8.675906980661981, 8.294773885233445, 8.273583516218238, 8.689987124182915, 9.139162216770474, + 8.822382059628058, 8.362948329273713, 8.443914703179315, 8.457037544672227, 8.17480477193586, + 7.965107052488668, 8.029678541545398 + ] + ] + } + } + ] + }, + "B": { + "status": 204 + } + } + } + \ No newline at end of file From bb0cd981b9c71f5da51ab4976e9a4fdd3dd86ee3 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 12:56:30 -0800 Subject: [PATCH 44/71] lint --- experimental/resource/openapi.go | 21 ++++++++++----------- experimental/resource/schema.go | 22 +++++++++++----------- experimental/resource/schema_test.go | 6 +++--- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/experimental/resource/openapi.go b/experimental/resource/openapi.go index 3369fb270..139d23267 100644 --- a/experimental/resource/openapi.go +++ b/experimental/resource/openapi.go @@ -3,8 +3,7 @@ package resource import ( "embed" - common "k8s.io/kube-openapi/pkg/common" - openapi "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/common" spec "k8s.io/kube-openapi/pkg/validation/spec" ) @@ -13,14 +12,14 @@ var f embed.FS func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ - "github.com/grafana/grafana-plugin-sdk-go/backend.QueryDataResponse": schema_backend_query_data_response(ref), - "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schema_data_frame(ref), - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.GenericDataQuery": schema_GenericQuery(ref), - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.QueryTypeDefinitionSpec": schema_QueryTypeDefinitionSpec(ref), + "github.com/grafana/grafana-plugin-sdk-go/backend.QueryDataResponse": schemaQueryDataResponse(ref), + "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.GenericDataQuery": schemaGenericQuery(ref), + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), } } -func schema_backend_query_data_response(_ common.ReferenceCallback) common.OpenAPIDefinition { +func schemaQueryDataResponse(_ common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -42,7 +41,7 @@ func schema_backend_query_data_response(_ common.ReferenceCallback) common.OpenA } } -func schema_data_frame(_ common.ReferenceCallback) common.OpenAPIDefinition { +func schemaDataFrame(_ common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -55,7 +54,7 @@ func schema_data_frame(_ common.ReferenceCallback) common.OpenAPIDefinition { } } -func schema_QueryTypeDefinitionSpec(_ common.ReferenceCallback) common.OpenAPIDefinition { +func schemaQueryTypeDefinitionSpec(_ common.ReferenceCallback) common.OpenAPIDefinition { s, _ := loadSchema("query.definition.schema.json") if s == nil { s = &spec.Schema{} @@ -65,14 +64,14 @@ func schema_QueryTypeDefinitionSpec(_ common.ReferenceCallback) common.OpenAPIDe } } -func schema_GenericQuery(_ common.ReferenceCallback) common.OpenAPIDefinition { +func schemaGenericQuery(_ common.ReferenceCallback) common.OpenAPIDefinition { s, _ := GenericQuerySchema() if s == nil { s = &spec.Schema{} } s.SchemaProps.Type = []string{"object"} s.SchemaProps.AdditionalProperties = &spec.SchemaOrBool{Allows: true} - return openapi.OpenAPIDefinition{Schema: *s} + return common.OpenAPIDefinition{Schema: *s} } // Get the cached feature list (exposed as a k8s resource) diff --git a/experimental/resource/schema.go b/experimental/resource/schema.go index 3d994850c..85d7b2fdd 100644 --- a/experimental/resource/schema.go +++ b/experimental/resource/schema.go @@ -22,10 +22,10 @@ func (s JSONSchema) MarshalJSON() ([]byte, error) { if err == nil { // The internal format puts $schema last! // this moves $schema first - copy := map[string]any{} - err := json.Unmarshal(body, ©) + cpy := map[string]any{} + err := json.Unmarshal(body, &cpy) if err == nil { - return json.Marshal(copy) + return json.Marshal(cpy) } } return body, err @@ -36,7 +36,7 @@ func (s *JSONSchema) UnmarshalJSON(data []byte) error { return s.Spec.UnmarshalJSON(data) } -func (g JSONSchema) OpenAPIDefinition() openapi.OpenAPIDefinition { +func (s JSONSchema) OpenAPIDefinition() openapi.OpenAPIDefinition { return openapi.OpenAPIDefinition{Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ Ref: spec.MustCreateRef(draft04), @@ -46,14 +46,14 @@ func (g JSONSchema) OpenAPIDefinition() openapi.OpenAPIDefinition { }} } -func (g *JSONSchema) DeepCopy() *JSONSchema { - if g == nil { +func (s *JSONSchema) DeepCopy() *JSONSchema { + if s == nil { return nil } out := &JSONSchema{} - if g.Spec != nil { + if s.Spec != nil { out.Spec = &spec.Schema{} - jj, err := json.Marshal(g.Spec) + jj, err := json.Marshal(s.Spec) if err == nil { _ = json.Unmarshal(jj, out.Spec) } @@ -61,10 +61,10 @@ func (g *JSONSchema) DeepCopy() *JSONSchema { return out } -func (g *JSONSchema) DeepCopyInto(out *JSONSchema) { - if g.Spec == nil { +func (s *JSONSchema) DeepCopyInto(out *JSONSchema) { + if s.Spec == nil { out.Spec = nil return } - out.Spec = g.DeepCopy().Spec + out.Spec = s.DeepCopy().Spec } diff --git a/experimental/resource/schema_test.go b/experimental/resource/schema_test.go index ea0b76ac4..061ac12d7 100644 --- a/experimental/resource/schema_test.go +++ b/experimental/resource/schema_test.go @@ -24,8 +24,8 @@ func TestSchemaSupport(t *testing.T) { fmt.Printf("%s\n", string(jj)) - copy := &JSONSchema{} - err = copy.UnmarshalJSON(jj) + cpy := &JSONSchema{} + err = cpy.UnmarshalJSON(jj) require.NoError(t, err) - require.Equal(t, val.Spec.Description, copy.Spec.Description) + require.Equal(t, val.Spec.Description, cpy.Spec.Description) } From 2b3cc4586303174b0f30b098fe97dcf9fcaefce8 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 16:05:10 -0800 Subject: [PATCH 45/71] now with unstructured --- experimental/resource/query_definition.go | 2 +- .../schemabuilder/example/query.types.json | 4 +- .../schemabuilder/example/query_test.go | 12 +-- .../resource/schemabuilder/examples.go | 22 ++--- .../resource/schemabuilder/reflector.go | 10 +-- experimental/resource/schemabuilder/schema.go | 14 ---- experimental/resource/unstructured.go | 81 +++++++++++++++++++ 7 files changed, 101 insertions(+), 44 deletions(-) create mode 100644 experimental/resource/unstructured.go diff --git a/experimental/resource/query_definition.go b/experimental/resource/query_definition.go index 132501456..88b1bb48f 100644 --- a/experimental/resource/query_definition.go +++ b/experimental/resource/query_definition.go @@ -68,7 +68,7 @@ type QueryExample struct { Description string `json:"description,omitempty"` // An example value saved that can be saved in a dashboard - SaveModel any `json:"saveModel,omitempty"` + SaveModel Unstructured `json:"saveModel,omitempty"` } type DiscriminatorFieldValue struct { diff --git a/experimental/resource/schemabuilder/example/query.types.json b/experimental/resource/schemabuilder/example/query.types.json index 24d5e03d4..cd5bb24f5 100644 --- a/experimental/resource/schemabuilder/example/query.types.json +++ b/experimental/resource/schemabuilder/example/query.types.json @@ -8,7 +8,7 @@ { "metadata": { "name": "math", - "resourceVersion": "1709230013217", + "resourceVersion": "1709250388761", "creationTimestamp": "2024-02-29T18:06:53Z" }, "spec": { @@ -56,7 +56,7 @@ { "metadata": { "name": "reduce", - "resourceVersion": "1709230013217", + "resourceVersion": "1709250388761", "creationTimestamp": "2024-02-29T18:06:53Z" }, "spec": { diff --git a/experimental/resource/schemabuilder/example/query_test.go b/experimental/resource/schemabuilder/example/query_test.go index 710e7b3c8..758b23dfe 100644 --- a/experimental/resource/schemabuilder/example/query_test.go +++ b/experimental/resource/schemabuilder/example/query_test.go @@ -28,15 +28,15 @@ func TestQueryTypeDefinitions(t *testing.T) { Examples: []resource.QueryExample{ { Name: "constant addition", - SaveModel: MathQuery{ + SaveModel: resource.AsUnstructured(MathQuery{ Expression: "$A + 10", - }, + }), }, { Name: "math with two queries", - SaveModel: MathQuery{ + SaveModel: resource.AsUnstructured(MathQuery{ Expression: "$A - $B", - }, + }), }, }, }, @@ -46,13 +46,13 @@ func TestQueryTypeDefinitions(t *testing.T) { Examples: []resource.QueryExample{ { Name: "get max value", - SaveModel: ReduceQuery{ + SaveModel: resource.AsUnstructured(ReduceQuery{ Expression: "$A", Reducer: ReducerMax, Settings: ReduceSettings{ Mode: ReduceModeDrop, }, - }, + }), }, }, }, diff --git a/experimental/resource/schemabuilder/examples.go b/experimental/resource/schemabuilder/examples.go index fe7261905..7162591cf 100644 --- a/experimental/resource/schemabuilder/examples.go +++ b/experimental/resource/schemabuilder/examples.go @@ -1,8 +1,6 @@ package schemabuilder import ( - "fmt" - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" ) @@ -15,12 +13,8 @@ func exampleRequest(defs resource.QueryTypeDefinitionList) (resource.GenericQuer for _, def := range defs.Items { for _, sample := range def.Spec.Examples { - if sample.SaveModel != nil { - q, err := asGenericDataQuery(sample.SaveModel) - if err != nil { - return rsp, fmt.Errorf("invalid sample save query [%s], in %s // %w", - sample.Name, def.ObjectMeta.Name, err) - } + if sample.SaveModel.Object != nil { + q := resource.NewGenericDataQuery(sample.SaveModel.Object) q.RefID = string(rune('A' + len(rsp.Queries))) for _, dis := range def.Spec.Discriminators { _ = q.Set(dis.Field, dis.Value) @@ -33,7 +27,7 @@ func exampleRequest(defs resource.QueryTypeDefinitionList) (resource.GenericQuer q.IntervalMS = 5 } - rsp.Queries = append(rsp.Queries, q) + rsp.Queries = append(rsp.Queries, &q) } } } @@ -45,18 +39,14 @@ func examplePanelTargets(ds *resource.DataSourceRef, defs resource.QueryTypeDefi for _, def := range defs.Items { for _, sample := range def.Spec.Examples { - if sample.SaveModel != nil { - q, err := asGenericDataQuery(sample.SaveModel) - if err != nil { - return nil, fmt.Errorf("invalid sample save query [%s], in %s // %w", - sample.Name, def.ObjectMeta.Name, err) - } + if sample.SaveModel.Object != nil { + q := resource.NewGenericDataQuery(sample.SaveModel.Object) q.Datasource = ds q.RefID = string(rune('A' + len(targets))) for _, dis := range def.Spec.Discriminators { _ = q.Set(dis.Field, dis.Value) } - targets = append(targets, *q) + targets = append(targets, q) } } } diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index c7b251372..571738e3c 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -9,6 +9,9 @@ import ( "testing" "time" + "github.com/go-openapi/jsonreference" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" "github.com/invopop/jsonschema" @@ -260,13 +263,10 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu defs.Items = append(defs.Items, def) } else { - b1, _ := json.Marshal(def.Spec) - b2, _ := json.Marshal(found.Spec) - if !assert.JSONEq(&testing.T{}, string(b1), string(b2)) { + if diff := cmp.Diff(def.Spec, found.Spec, cmpopts.IgnoreUnexported(jsonreference.Ref{})); diff != "" { + fmt.Printf("Spec changed:\n%s\n", diff) found.ObjectMeta.ResourceVersion = rv found.Spec = def.Spec - - fmt.Printf("NEW:%s\n", string(b2)) } delete(byName, def.ObjectMeta.Name) } diff --git a/experimental/resource/schemabuilder/schema.go b/experimental/resource/schemabuilder/schema.go index e13194ab3..68c62ecd7 100644 --- a/experimental/resource/schemabuilder/schema.go +++ b/experimental/resource/schemabuilder/schema.go @@ -188,17 +188,3 @@ func asJSONSchema(v any) (*spec.Schema, error) { err = json.Unmarshal(b, s) return s, err } - -func asGenericDataQuery(v any) (*resource.GenericDataQuery, error) { - s, ok := v.(*resource.GenericDataQuery) - if ok { - return s, nil - } - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - s = &resource.GenericDataQuery{} - err = json.Unmarshal(b, s) - return s, err -} diff --git a/experimental/resource/unstructured.go b/experimental/resource/unstructured.go new file mode 100644 index 000000000..4e52c242d --- /dev/null +++ b/experimental/resource/unstructured.go @@ -0,0 +1,81 @@ +package resource + +import ( + "encoding/json" + + openapi "k8s.io/kube-openapi/pkg/common" + spec "k8s.io/kube-openapi/pkg/validation/spec" +) + +// Unstructured allows objects that do not have Golang structs registered to be manipulated +// generically. +type Unstructured struct { + // Object is a JSON compatible map with string, float, int, bool, []interface{}, + // or map[string]interface{} children. + Object map[string]any +} + +// Create an unstructured value from any input +func AsUnstructured(v any) Unstructured { + out := Unstructured{} + body, err := json.Marshal(v) + if err == nil { + _ = json.Unmarshal(body, &out.Object) + } + return out +} + +// Produce an API definition that represents map[string]any +func (u Unstructured) OpenAPIDefinition() openapi.OpenAPIDefinition { + return openapi.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + }, + } +} + +func (u *Unstructured) UnstructuredContent() map[string]interface{} { + if u.Object == nil { + return make(map[string]interface{}) + } + return u.Object +} + +func (u *Unstructured) SetUnstructuredContent(content map[string]interface{}) { + u.Object = content +} + +// MarshalJSON ensures that the unstructured object produces proper +// JSON when passed to Go's standard JSON library. +func (u *Unstructured) MarshalJSON() ([]byte, error) { + return json.Marshal(u.Object) +} + +// UnmarshalJSON ensures that the unstructured object properly decodes +// JSON when passed to Go's standard JSON library. +func (u *Unstructured) UnmarshalJSON(b []byte) error { + return json.Unmarshal(b, &u.Object) +} + +func (u *Unstructured) DeepCopy() *Unstructured { + if u == nil { + return nil + } + out := new(Unstructured) + u.DeepCopyInto(out) + return out +} + +func (u *Unstructured) DeepCopyInto(out *Unstructured) { + obj := map[string]any{} + if u.Object != nil { + jj, err := json.Marshal(u.Object) + if err == nil { + _ = json.Unmarshal(jj, &obj) + } + } + out.Object = obj +} From 0b0e1ea7adf7ae28190aa8ad338d1cc0f828287e Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 16:18:42 -0800 Subject: [PATCH 46/71] less verbose --- experimental/resource/query_definition.go | 2 +- .../schemabuilder/example/query.panel.example.json | 8 ++++---- .../schemabuilder/example/query.request.example.json | 8 ++++---- .../resource/schemabuilder/example/query.types.json | 9 +++++---- .../resource/schemabuilder/example/query_test.go | 3 ++- experimental/resource/schemabuilder/reflector.go | 2 +- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/experimental/resource/query_definition.go b/experimental/resource/query_definition.go index 88b1bb48f..32203efc2 100644 --- a/experimental/resource/query_definition.go +++ b/experimental/resource/query_definition.go @@ -35,7 +35,7 @@ type QueryTypeDefinitionSpec struct { Schema JSONSchema `json:"schema"` // Examples (include a wrapper) ideally a template! - Examples []QueryExample `json:"examples,omitempty"` + Examples []QueryExample `json:"examples"` // Changelog defines the changed from the previous version // All changes in the same version *must* be backwards compatible diff --git a/experimental/resource/schemabuilder/example/query.panel.example.json b/experimental/resource/schemabuilder/example/query.panel.example.json index c02694bb2..84dab155f 100644 --- a/experimental/resource/schemabuilder/example/query.panel.example.json +++ b/experimental/resource/schemabuilder/example/query.panel.example.json @@ -8,7 +8,7 @@ "uid": "TheUID" }, "queryType": "math", - "expression": "$A + 10" + "expression": "$A + 11" }, { "refId": "B", @@ -26,11 +26,11 @@ "uid": "TheUID" }, "queryType": "reduce", - "expression": "$A", - "reducer": "max", "settings": { "mode": "dropNN" - } + }, + "expression": "$A", + "reducer": "max" } ] } \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.request.example.json b/experimental/resource/schemabuilder/example/query.request.example.json index 871d54dc8..e48a2f930 100644 --- a/experimental/resource/schemabuilder/example/query.request.example.json +++ b/experimental/resource/schemabuilder/example/query.request.example.json @@ -7,7 +7,7 @@ "queryType": "math", "maxDataPoints": 1000, "intervalMs": 5, - "expression": "$A + 10" + "expression": "$A + 11" }, { "refId": "B", @@ -21,11 +21,11 @@ "queryType": "reduce", "maxDataPoints": 1000, "intervalMs": 5, - "expression": "$A", - "reducer": "max", "settings": { "mode": "dropNN" - } + }, + "expression": "$A", + "reducer": "max" } ] } \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.types.json b/experimental/resource/schemabuilder/example/query.types.json index cd5bb24f5..570c02ab0 100644 --- a/experimental/resource/schemabuilder/example/query.types.json +++ b/experimental/resource/schemabuilder/example/query.types.json @@ -8,7 +8,7 @@ { "metadata": { "name": "math", - "resourceVersion": "1709250388761", + "resourceVersion": "1709251645142", "creationTimestamp": "2024-02-29T18:06:53Z" }, "spec": { @@ -41,7 +41,7 @@ { "name": "constant addition", "saveModel": { - "expression": "$A + 10" + "expression": "$A + 11" } }, { @@ -141,7 +141,7 @@ { "metadata": { "name": "resample", - "resourceVersion": "1709230013217", + "resourceVersion": "1709252275481", "creationTimestamp": "2024-02-29T18:06:53Z" }, "spec": { @@ -186,7 +186,8 @@ "loadedDimensions" ], "type": "object" - } + }, + "examples": [] } } ] diff --git a/experimental/resource/schemabuilder/example/query_test.go b/experimental/resource/schemabuilder/example/query_test.go index 758b23dfe..0a9e484ca 100644 --- a/experimental/resource/schemabuilder/example/query_test.go +++ b/experimental/resource/schemabuilder/example/query_test.go @@ -29,7 +29,7 @@ func TestQueryTypeDefinitions(t *testing.T) { { Name: "constant addition", SaveModel: resource.AsUnstructured(MathQuery{ - Expression: "$A + 10", + Expression: "$A + 11", }), }, { @@ -59,6 +59,7 @@ func TestQueryTypeDefinitions(t *testing.T) { schemabuilder.QueryTypeInfo{ Discriminators: resource.NewDiscriminators("queryType", QueryTypeResample), GoType: reflect.TypeOf(&ResampleQuery{}), + Examples: []resource.QueryExample{}, }) require.NoError(t, err) diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index 571738e3c..6a2931e48 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -264,7 +264,7 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu defs.Items = append(defs.Items, def) } else { if diff := cmp.Diff(def.Spec, found.Spec, cmpopts.IgnoreUnexported(jsonreference.Ref{})); diff != "" { - fmt.Printf("Spec changed:\n%s\n", diff) + // fmt.Printf("Spec changed:\n%s\n", diff) found.ObjectMeta.ResourceVersion = rv found.Spec = def.Spec } From a804614412636ae6a026036e4fbe541074bd0f7d Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 19:21:21 -0800 Subject: [PATCH 47/71] now with unstructured --- .../resource/query.definition.schema.json | 5 +++- .../resource/schemabuilder/reflector.go | 25 ++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/experimental/resource/query.definition.schema.json b/experimental/resource/query.definition.schema.json index e23f11d3c..d71d2569c 100644 --- a/experimental/resource/query.definition.schema.json +++ b/experimental/resource/query.definition.schema.json @@ -44,6 +44,8 @@ "description": "Optionally explain why the example is interesting" }, "saveModel": { + "additionalProperties": true, + "type": "object", "description": "An example value saved that can be saved in a dashboard" } }, @@ -64,6 +66,7 @@ "additionalProperties": false, "type": "object", "required": [ - "schema" + "schema", + "examples" ] } \ No newline at end of file diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index 6a2931e48..494cee037 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/go-openapi/jsonreference" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/grafana/grafana-plugin-sdk-go/data" @@ -95,6 +94,10 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { }, AdditionalProperties: jsonschema.TrueSchema, }, + reflect.TypeOf(resource.Unstructured{}): { + Type: "object", + AdditionalProperties: jsonschema.TrueSchema, + }, reflect.TypeOf(resource.JSONSchema{}): { Type: "object", Ref: draft04, @@ -263,8 +266,10 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu defs.Items = append(defs.Items, def) } else { - if diff := cmp.Diff(def.Spec, found.Spec, cmpopts.IgnoreUnexported(jsonreference.Ref{})); diff != "" { - // fmt.Printf("Spec changed:\n%s\n", diff) + x := resource.AsUnstructured(def.Spec) + y := resource.AsUnstructured(found.Spec) + if diff := cmp.Diff(stripNilValues(x.Object), stripNilValues(y.Object), cmpopts.EquateEmpty()); diff != "" { + fmt.Printf("Spec changed:\n%s\n", diff) found.ObjectMeta.ResourceVersion = rv found.Spec = def.Spec } @@ -422,3 +427,17 @@ func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { require.NoError(t, err, "error writing file") } } + +func stripNilValues(input map[string]any) map[string]any { + for k, v := range input { + if v == nil { + delete(input, k) + } else { + sub, ok := v.(map[string]any) + if ok { + stripNilValues(sub) + } + } + } + return input +} From 424be9acc75fdce74487a26250249e1fc775bc8f Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 29 Feb 2024 19:25:06 -0800 Subject: [PATCH 48/71] now with unstructured --- experimental/resource/schemabuilder/examples.go | 4 ++-- experimental/resource/schemabuilder/reflector.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/experimental/resource/schemabuilder/examples.go b/experimental/resource/schemabuilder/examples.go index 7162591cf..7c5f6a8d6 100644 --- a/experimental/resource/schemabuilder/examples.go +++ b/experimental/resource/schemabuilder/examples.go @@ -4,7 +4,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" ) -func exampleRequest(defs resource.QueryTypeDefinitionList) (resource.GenericQueryRequest, error) { +func exampleRequest(defs resource.QueryTypeDefinitionList) resource.GenericQueryRequest { rsp := resource.GenericQueryRequest{ From: "now-1h", To: "now", @@ -31,7 +31,7 @@ func exampleRequest(defs resource.QueryTypeDefinitionList) (resource.GenericQuer } } } - return rsp, nil + return rsp } func examplePanelTargets(ds *resource.DataSourceRef, defs resource.QueryTypeDefinitionList) ([]resource.GenericDataQuery, error) { diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index 494cee037..70099a8b4 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -325,9 +325,7 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, schema, body) - request, err := exampleRequest(defs) - require.NoError(t, err) - + request := exampleRequest(defs) outfile = filepath.Join(outdir, "query.request.example.json") body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, request, body) From 3a97c127130fda1f06f49478a91d6024b78c1997 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 1 Mar 2024 13:33:31 -0800 Subject: [PATCH 49/71] more copy methods --- backend/data.go | 39 +++++++++++++++++++++++++++++++- experimental/resource/openapi.go | 19 ++++------------ 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/backend/data.go b/backend/data.go index 697b4e516..c7a518238 100644 --- a/backend/data.go +++ b/backend/data.go @@ -111,7 +111,7 @@ type DataQuery struct { // It is the return type of a QueryData call. type QueryDataResponse struct { // Responses is a map of RefIDs (Unique Query ID) to *DataResponse. - Responses Responses + Responses Responses `json:"results"` } // MarshalJSON writes the results as json @@ -131,6 +131,30 @@ func (r *QueryDataResponse) UnmarshalJSON(b []byte) error { return iter.Error } +func (g *QueryDataResponse) DeepCopy() *QueryDataResponse { + if g == nil { + return nil + } + out := new(QueryDataResponse) + g.DeepCopyInto(out) + return out +} + +func (g *QueryDataResponse) DeepCopyInto(out *QueryDataResponse) { + if g.Responses == nil { + out.Responses = nil + return + } + if out.Responses == nil { + out.Responses = make(Responses, len(g.Responses)) + } else { + clear(out.Responses) + } + for k, v := range g.Responses { + out.Responses[k] = *v.DeepCopy() + } +} + // NewQueryDataResponse returns a QueryDataResponse with the Responses property initialized. func NewQueryDataResponse() *QueryDataResponse { return &QueryDataResponse{ @@ -191,6 +215,19 @@ func (r DataResponse) MarshalJSON() ([]byte, error) { return append([]byte(nil), stream.Buffer()...), stream.Error } +func (r *DataResponse) DeepCopy() *DataResponse { + if r == nil { + return nil + } + out := &DataResponse{} + body, err := r.MarshalJSON() + if err == nil { + iter := jsoniter.ParseBytes(jsoniter.ConfigDefault, body) + readDataResponseJSON(out, iter) + } + return out +} + // TimeRange represents a time range for a query and is a property of DataQuery. type TimeRange struct { // From is the start time of the query. diff --git a/experimental/resource/openapi.go b/experimental/resource/openapi.go index 139d23267..f9af8353f 100644 --- a/experimental/resource/openapi.go +++ b/experimental/resource/openapi.go @@ -12,29 +12,20 @@ var f embed.FS func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ - "github.com/grafana/grafana-plugin-sdk-go/backend.QueryDataResponse": schemaQueryDataResponse(ref), + "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.GenericDataQuery": schemaGenericQuery(ref), "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), } } -func schemaQueryDataResponse(_ common.ReferenceCallback) common.OpenAPIDefinition { +// Individual response +func schemaDataResponse(_ common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "results keyed by refId", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "results": *spec.MapProperty(&spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "any object for now", - Type: []string{"object"}, - Properties: map[string]spec.Schema{}, - AdditionalProperties: &spec.SchemaOrBool{Allows: true}, - }, - }), - }, + Description: "todo... improve schema", + Type: []string{"object"}, AdditionalProperties: &spec.SchemaOrBool{Allows: false}, }, }, From f8336e6d4667ef7e8a59156ff8528a65cb230f9f Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 1 Mar 2024 23:33:07 -0800 Subject: [PATCH 50/71] fix lint remove parser --- backend/data.go | 12 +-- experimental/resource/openapi.go | 2 +- experimental/resource/openapi_test.go | 2 +- experimental/resource/panel.go | 8 +- experimental/resource/query.go | 59 +++++------ experimental/resource/query_parser.go | 98 ------------------- experimental/resource/query_test.go | 15 ++- .../resource/schemabuilder/examples.go | 18 ++-- .../resource/schemabuilder/reflector.go | 5 +- .../testdata/sample_query_results.json | 92 ++++++++--------- 10 files changed, 94 insertions(+), 217 deletions(-) delete mode 100644 experimental/resource/query_parser.go diff --git a/backend/data.go b/backend/data.go index c7a518238..e67ae4c8d 100644 --- a/backend/data.go +++ b/backend/data.go @@ -131,26 +131,26 @@ func (r *QueryDataResponse) UnmarshalJSON(b []byte) error { return iter.Error } -func (g *QueryDataResponse) DeepCopy() *QueryDataResponse { +func (r *QueryDataResponse) DeepCopy() *QueryDataResponse { if g == nil { return nil } out := new(QueryDataResponse) - g.DeepCopyInto(out) + r.DeepCopyInto(out) return out } -func (g *QueryDataResponse) DeepCopyInto(out *QueryDataResponse) { - if g.Responses == nil { +func (r *QueryDataResponse) DeepCopyInto(out *QueryDataResponse) { + if r.Responses == nil { out.Responses = nil return } if out.Responses == nil { - out.Responses = make(Responses, len(g.Responses)) + out.Responses = make(Responses, len(r.Responses)) } else { clear(out.Responses) } - for k, v := range g.Responses { + for k, v := range r.Responses { out.Responses[k] = *v.DeepCopy() } } diff --git a/experimental/resource/openapi.go b/experimental/resource/openapi.go index f9af8353f..d5ce09f09 100644 --- a/experimental/resource/openapi.go +++ b/experimental/resource/openapi.go @@ -26,7 +26,7 @@ func schemaDataResponse(_ common.ReferenceCallback) common.OpenAPIDefinition { SchemaProps: spec.SchemaProps{ Description: "todo... improve schema", Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{Allows: false}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, }, }, } diff --git a/experimental/resource/openapi_test.go b/experimental/resource/openapi_test.go index 74def6589..d3dfe12af 100644 --- a/experimental/resource/openapi_test.go +++ b/experimental/resource/openapi_test.go @@ -18,7 +18,7 @@ func TestOpenAPI(t *testing.T) { return spec.MustCreateRef(path) // placeholder for tests }) - def, ok := defs["github.com/grafana/grafana-plugin-sdk-go/backend.QueryDataResponse"] + def, ok := defs["github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse"] require.True(t, ok) require.Empty(t, def.Dependencies) // not yet supported! diff --git a/experimental/resource/panel.go b/experimental/resource/panel.go index f2a426a2d..e107f7dbe 100644 --- a/experimental/resource/panel.go +++ b/experimental/resource/panel.go @@ -1,6 +1,6 @@ package resource -type PseudoPanel[Q any] struct { +type PseudoPanel struct { // Numeric panel id ID int `json:"id,omitempty"` @@ -11,14 +11,14 @@ type PseudoPanel[Q any] struct { Title string `json:"title,omitempty"` // Options depend on the panel type - Options map[string]any `json:"options,omitempty"` + Options Unstructured `json:"options,omitempty"` // FieldConfig values depend on the panel type - FieldConfig map[string]any `json:"fieldConfig,omitempty"` + FieldConfig Unstructured `json:"fieldConfig,omitempty"` // This should no longer be necessary since each target has the datasource reference Datasource *DataSourceRef `json:"datasource,omitempty"` // The query targets - Targets []Q `json:"targets"` + Targets []DataQuery `json:"targets"` } diff --git a/experimental/resource/query.go b/experimental/resource/query.go index 5edea77bc..2e862df38 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -12,19 +12,11 @@ import ( ) func init() { //nolint:gochecknoinits - jsoniter.RegisterTypeEncoder("resource.GenericDataQuery", &genericQueryCodec{}) - jsoniter.RegisterTypeDecoder("resource.GenericDataQuery", &genericQueryCodec{}) + jsoniter.RegisterTypeEncoder("resource.DataQuery", &genericQueryCodec{}) + jsoniter.RegisterTypeDecoder("resource.DataQuery", &genericQueryCodec{}) } -type DataQuery interface { - // The standard query properties - CommonProperties() *CommonQueryProperties - - // For queries that depend on other queries to run first (eg, other refIds) - Dependencies() []string -} - -type QueryRequest[Q DataQuery] struct { +type DataQueryRequest struct { // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. // example: now-1h From string `json:"from,omitempty"` @@ -34,19 +26,14 @@ type QueryRequest[Q DataQuery] struct { To string `json:"to,omitempty"` // Each item has a - Queries []Q `json:"queries"` + Queries []DataQuery `json:"queries"` // required: false Debug bool `json:"debug,omitempty"` } -// GenericQueryRequest is a query request that supports any datasource -type GenericQueryRequest = QueryRequest[*GenericDataQuery] - -var _ DataQuery = (*GenericDataQuery)(nil) - -// GenericDataQuery is a replacement for `dtos.MetricRequest` with more explicit typing -type GenericDataQuery struct { +// DataQuery is a replacement for `dtos.MetricRequest` with more explicit typing +type DataQuery struct { CommonQueryProperties `json:",inline"` // Additional Properties (that live at the root) @@ -54,17 +41,17 @@ type GenericDataQuery struct { } // CommonProperties implements DataQuery. -func (g *GenericDataQuery) CommonProperties() *CommonQueryProperties { +func (g *DataQuery) CommonProperties() *CommonQueryProperties { return &g.CommonQueryProperties } // Dependencies implements DataQuery. -func (g *GenericDataQuery) Dependencies() []string { +func (g *DataQuery) Dependencies() []string { return nil } -func NewGenericDataQuery(body map[string]any) GenericDataQuery { - g := &GenericDataQuery{ +func NewDataQuery(body map[string]any) DataQuery { + g := &DataQuery{ additional: make(map[string]any), } for k, v := range body { @@ -73,11 +60,11 @@ func NewGenericDataQuery(body map[string]any) GenericDataQuery { return *g } -func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { +func (g *DataQuery) DeepCopy() *DataQuery { if g == nil { return nil } - out := new(GenericDataQuery) + out := new(DataQuery) jj, err := json.Marshal(g) if err == nil { _ = json.Unmarshal(jj, out) @@ -85,13 +72,13 @@ func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { return out } -func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { +func (g *DataQuery) DeepCopyInto(out *DataQuery) { clone := g.DeepCopy() *out = *clone } // Set allows setting values using key/value pairs -func (g *GenericDataQuery) Set(key string, val any) *GenericDataQuery { +func (g *DataQuery) Set(key string, val any) *DataQuery { switch key { case "refId": g.RefID, _ = val.(string) @@ -138,7 +125,7 @@ func (g *GenericDataQuery) Set(key string, val any) *GenericDataQuery { return g } -func (g *GenericDataQuery) Get(key string) (any, bool) { +func (g *DataQuery) Get(key string) (any, bool) { switch key { case "refId": return g.RefID, true @@ -163,7 +150,7 @@ func (g *GenericDataQuery) Get(key string) (any, bool) { return v, ok } -func (g *GenericDataQuery) MustString(key string) string { +func (g *DataQuery) MustString(key string) string { v, ok := g.Get(key) if ok { s, ok := v.(string) @@ -181,12 +168,12 @@ func (codec *genericQueryCodec) IsEmpty(_ unsafe.Pointer) bool { } func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { - q := (*GenericDataQuery)(ptr) + q := (*DataQuery)(ptr) writeQuery(q, stream) } func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { - q := GenericDataQuery{} + q := DataQuery{} err := q.readQuery(jsoniter.NewIterator(iter)) if err != nil { // keep existing iter error if it exists @@ -195,11 +182,11 @@ func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { } return } - *((*GenericDataQuery)(ptr)) = q + *((*DataQuery)(ptr)) = q } // MarshalJSON writes JSON including the common and custom values -func (g GenericDataQuery) MarshalJSON() ([]byte, error) { +func (g DataQuery) MarshalJSON() ([]byte, error) { cfg := j.ConfigCompatibleWithStandardLibrary stream := cfg.BorrowStream(nil) defer cfg.ReturnStream(stream) @@ -209,7 +196,7 @@ func (g GenericDataQuery) MarshalJSON() ([]byte, error) { } // UnmarshalJSON reads a query from json byte array -func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { +func (g *DataQuery) UnmarshalJSON(b []byte) error { iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, b) if err != nil { return err @@ -217,7 +204,7 @@ func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { return g.readQuery(iter) } -func writeQuery(g *GenericDataQuery, stream *j.Stream) { +func writeQuery(g *DataQuery, stream *j.Stream) { q := g.CommonQueryProperties stream.WriteObjectStart() stream.WriteObjectField("refId") @@ -282,7 +269,7 @@ func writeQuery(g *GenericDataQuery, stream *j.Stream) { stream.WriteObjectEnd() } -func (g *GenericDataQuery) readQuery(iter *jsoniter.Iterator) error { +func (g *DataQuery) readQuery(iter *jsoniter.Iterator) error { return g.CommonQueryProperties.readQuery(iter, func(key string, iter *jsoniter.Iterator) error { if g.additional == nil { g.additional = make(map[string]any) diff --git a/experimental/resource/query_parser.go b/experimental/resource/query_parser.go deleted file mode 100644 index a21c7a656..000000000 --- a/experimental/resource/query_parser.go +++ /dev/null @@ -1,98 +0,0 @@ -package resource - -import ( - "time" - - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" -) - -func ParseQueryRequest(iter *jsoniter.Iterator) (*GenericQueryRequest, error) { - return ParseTypedQueryRequest[*GenericDataQuery](&genericQueryReader{}, iter, time.Now()) -} - -type TypedQueryReader[T DataQuery] interface { - // Called before any custom property is found - Start(p *CommonQueryProperties, now time.Time) error - // Called for each non-common property - SetProperty(key string, iter *jsoniter.Iterator) error - // Finished reading the JSON node - Finish() (T, error) -} - -func ParseTypedQueryRequest[T DataQuery](reader TypedQueryReader[T], iter *jsoniter.Iterator, now time.Time) (*QueryRequest[T], error) { - var err error - var root string - var ok bool - dqr := &QueryRequest[T]{} - for root, err = iter.ReadObject(); root != ""; root, err = iter.ReadObject() { - switch root { - case "to": - dqr.To, err = iter.ReadString() - case "from": - dqr.From, err = iter.ReadString() - case "debug": - dqr.Debug, err = iter.ReadBool() - case "queries": - ok, err = iter.ReadArray() - for ok && err == nil { - props := &CommonQueryProperties{} - err = reader.Start(props, now) - if err != nil { - return dqr, err - } - err = props.readQuery(iter, reader.SetProperty) - if err != nil { - return dqr, err - } - - q, err := reader.Finish() - if err != nil { - return dqr, err - } - dqr.Queries = append(dqr.Queries, q) - - ok, err = iter.ReadArray() - if err != nil { - return dqr, err - } - } - default: - // ignored? or error - } - if err != nil { - return dqr, err - } - } - return dqr, err -} - -var _ TypedQueryReader[*GenericDataQuery] = (*genericQueryReader)(nil) - -type genericQueryReader struct { - common *CommonQueryProperties - additional map[string]any -} - -// Called before any custom properties are passed -func (g *genericQueryReader) Start(p *CommonQueryProperties, _ time.Time) error { - g.additional = make(map[string]any) - g.common = p - return nil -} - -func (g *genericQueryReader) SetProperty(key string, iter *jsoniter.Iterator) error { - v, err := iter.Read() - if err != nil { - return err - } - g.additional[key] = v - return nil -} - -// Finished the JSON node, return a query object -func (g *genericQueryReader) Finish() (*GenericDataQuery, error) { - return &GenericDataQuery{ - CommonQueryProperties: *g.common, - additional: g.additional, - }, nil -} diff --git a/experimental/resource/query_test.go b/experimental/resource/query_test.go index c349ee9fb..41877fb61 100644 --- a/experimental/resource/query_test.go +++ b/experimental/resource/query_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "testing" - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" "github.com/stretchr/testify/require" ) @@ -37,7 +36,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { "to": "1692646267389" }`) - req := &GenericQueryRequest{} + req := &DataQueryRequest{} err := json.Unmarshal(request, req) require.NoError(t, err) @@ -51,7 +50,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { require.NoError(t, err) // And read it back with standard JSON marshal functions - query := &GenericDataQuery{} + query := &DataQuery{} err = json.Unmarshal(out, query) require.NoError(t, err) require.Equal(t, "spreadsheetID", query.MustString("spreadsheet")) @@ -75,10 +74,8 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { }) t.Run("same results from either parser", func(t *testing.T) { - iter, err := jsoniter.ParseBytes(jsoniter.ConfigCompatibleWithStandardLibrary, request) - require.NoError(t, err) - - typed, err := ParseQueryRequest(iter) + typed := &DataQueryRequest{} + err = json.Unmarshal(request, typed) require.NoError(t, err) out1, err := json.MarshalIndent(req, "", " ") @@ -93,7 +90,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { func TestQueryBuilders(t *testing.T) { prop := "testkey" - testQ1 := &GenericDataQuery{} + testQ1 := &DataQuery{} testQ1.Set(prop, "A") require.Equal(t, "A", testQ1.MustString(prop)) @@ -112,7 +109,7 @@ func TestQueryBuilders(t *testing.T) { require.Equal(t, "D", testQ2.MustString("queryType")) // Map constructor - testQ3 := NewGenericDataQuery(map[string]any{ + testQ3 := NewDataQuery(map[string]any{ "queryType": "D", "extra": "E", }) diff --git a/experimental/resource/schemabuilder/examples.go b/experimental/resource/schemabuilder/examples.go index 7c5f6a8d6..8654c50ed 100644 --- a/experimental/resource/schemabuilder/examples.go +++ b/experimental/resource/schemabuilder/examples.go @@ -4,17 +4,17 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" ) -func exampleRequest(defs resource.QueryTypeDefinitionList) resource.GenericQueryRequest { - rsp := resource.GenericQueryRequest{ +func exampleRequest(defs resource.QueryTypeDefinitionList) resource.DataQueryRequest { + rsp := resource.DataQueryRequest{ From: "now-1h", To: "now", - Queries: []*resource.GenericDataQuery{}, + Queries: []resource.DataQuery{}, } for _, def := range defs.Items { for _, sample := range def.Spec.Examples { if sample.SaveModel.Object != nil { - q := resource.NewGenericDataQuery(sample.SaveModel.Object) + q := resource.NewDataQuery(sample.SaveModel.Object) q.RefID = string(rune('A' + len(rsp.Queries))) for _, dis := range def.Spec.Discriminators { _ = q.Set(dis.Field, dis.Value) @@ -27,20 +27,20 @@ func exampleRequest(defs resource.QueryTypeDefinitionList) resource.GenericQuery q.IntervalMS = 5 } - rsp.Queries = append(rsp.Queries, &q) + rsp.Queries = append(rsp.Queries, q) } } } return rsp } -func examplePanelTargets(ds *resource.DataSourceRef, defs resource.QueryTypeDefinitionList) ([]resource.GenericDataQuery, error) { - targets := []resource.GenericDataQuery{} +func examplePanelTargets(ds *resource.DataSourceRef, defs resource.QueryTypeDefinitionList) []resource.DataQuery { + targets := []resource.DataQuery{} for _, def := range defs.Items { for _, sample := range def.Spec.Examples { if sample.SaveModel.Object != nil { - q := resource.NewGenericDataQuery(sample.SaveModel.Object) + q := resource.NewDataQuery(sample.SaveModel.Object) q.Datasource = ds q.RefID = string(rune('A' + len(targets))) for _, dis := range def.Spec.Discriminators { @@ -50,5 +50,5 @@ func examplePanelTargets(ds *resource.DataSourceRef, defs resource.QueryTypeDefi } } } - return targets, nil + return targets } diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/resource/schemabuilder/reflector.go index 70099a8b4..91db27556 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/resource/schemabuilder/reflector.go @@ -299,14 +299,13 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, schema, body) - panel := resource.PseudoPanel[resource.GenericDataQuery]{ + panel := resource.PseudoPanel{ Type: "table", } - panel.Targets, err = examplePanelTargets(&resource.DataSourceRef{ + panel.Targets = examplePanelTargets(&resource.DataSourceRef{ Type: b.opts.PluginID[0], UID: "TheUID", }, defs) - require.NoError(t, err) outfile = filepath.Join(outdir, "query.panel.example.json") body, _ = os.ReadFile(outfile) diff --git a/experimental/resource/testdata/sample_query_results.json b/experimental/resource/testdata/sample_query_results.json index 30de119d4..4d0fd14f5 100644 --- a/experimental/resource/testdata/sample_query_results.json +++ b/experimental/resource/testdata/sample_query_results.json @@ -1,59 +1,51 @@ { - "results": { - "A": { - "status": 200, - "frames": [ + "status": 200, + "frames": [ + { + "schema": { + "refId": "A", + "meta": { + "typeVersion": [0, 0], + "custom": { + "customStat": 10 + } + }, + "fields": [ { - "schema": { - "refId": "A", - "meta": { - "typeVersion": [0, 0], - "custom": { - "customStat": 10 - } - }, - "fields": [ - { - "name": "time", - "type": "time", - "typeInfo": { - "frame": "time.Time", - "nullable": true - }, - "config": { - "interval": 1800000 - } - }, - { - "name": "A-series", - "type": "number", - "typeInfo": { - "frame": "float64", - "nullable": true - }, - "labels": {} - } - ] + "name": "time", + "type": "time", + "typeInfo": { + "frame": "time.Time", + "nullable": true }, - "data": { - "values": [ - [ - 1708955198367, 1708956998367, 1708958798367, 1708960598367, 1708962398367, 1708964198367, 1708965998367, - 1708967798367, 1708969598367, 1708971398367, 1708973198367, 1708974998367 - ], - [ - 8.675906980661981, 8.294773885233445, 8.273583516218238, 8.689987124182915, 9.139162216770474, - 8.822382059628058, 8.362948329273713, 8.443914703179315, 8.457037544672227, 8.17480477193586, - 7.965107052488668, 8.029678541545398 - ] - ] + "config": { + "interval": 1800000 } + }, + { + "name": "A-series", + "type": "number", + "typeInfo": { + "frame": "float64", + "nullable": true + }, + "labels": {} } ] }, - "B": { - "status": 204 + "data": { + "values": [ + [ + 1708955198367, 1708956998367, 1708958798367, 1708960598367, 1708962398367, 1708964198367, 1708965998367, + 1708967798367, 1708969598367, 1708971398367, 1708973198367, 1708974998367 + ], + [ + 8.675906980661981, 8.294773885233445, 8.273583516218238, 8.689987124182915, 9.139162216770474, + 8.822382059628058, 8.362948329273713, 8.443914703179315, 8.457037544672227, 8.17480477193586, + 7.965107052488668, 8.029678541545398 + ] + ] } } - } - \ No newline at end of file + ] +} From 2a7ba037cbdabcf0bfdc912d844a86ab275b865c Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 1 Mar 2024 23:40:35 -0800 Subject: [PATCH 51/71] fix lint --- backend/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/data.go b/backend/data.go index e67ae4c8d..fc6fbe7cd 100644 --- a/backend/data.go +++ b/backend/data.go @@ -132,7 +132,7 @@ func (r *QueryDataResponse) UnmarshalJSON(b []byte) error { } func (r *QueryDataResponse) DeepCopy() *QueryDataResponse { - if g == nil { + if r == nil { return nil } out := new(QueryDataResponse) From 2ec14716ff3664d26cb8b99892300094258caedf Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 1 Mar 2024 23:58:16 -0800 Subject: [PATCH 52/71] panel cleanup --- experimental/resource/panel.go | 6 ------ .../resource/schemabuilder/example/query.panel.example.json | 6 +++--- experimental/resource/unstructured.go | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/experimental/resource/panel.go b/experimental/resource/panel.go index e107f7dbe..252f594f5 100644 --- a/experimental/resource/panel.go +++ b/experimental/resource/panel.go @@ -10,12 +10,6 @@ type PseudoPanel struct { // The panel title Title string `json:"title,omitempty"` - // Options depend on the panel type - Options Unstructured `json:"options,omitempty"` - - // FieldConfig values depend on the panel type - FieldConfig Unstructured `json:"fieldConfig,omitempty"` - // This should no longer be necessary since each target has the datasource reference Datasource *DataSourceRef `json:"datasource,omitempty"` diff --git a/experimental/resource/schemabuilder/example/query.panel.example.json b/experimental/resource/schemabuilder/example/query.panel.example.json index 84dab155f..5999e8854 100644 --- a/experimental/resource/schemabuilder/example/query.panel.example.json +++ b/experimental/resource/schemabuilder/example/query.panel.example.json @@ -26,11 +26,11 @@ "uid": "TheUID" }, "queryType": "reduce", + "expression": "$A", + "reducer": "max", "settings": { "mode": "dropNN" - }, - "expression": "$A", - "reducer": "max" + } } ] } \ No newline at end of file diff --git a/experimental/resource/unstructured.go b/experimental/resource/unstructured.go index 4e52c242d..625fdd848 100644 --- a/experimental/resource/unstructured.go +++ b/experimental/resource/unstructured.go @@ -50,7 +50,7 @@ func (u *Unstructured) SetUnstructuredContent(content map[string]interface{}) { // MarshalJSON ensures that the unstructured object produces proper // JSON when passed to Go's standard JSON library. -func (u *Unstructured) MarshalJSON() ([]byte, error) { +func (u Unstructured) MarshalJSON() ([]byte, error) { return json.Marshal(u.Object) } From 0237a4034264addc38a7f78a16cb9a3722ade4d8 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 1 Mar 2024 23:59:36 -0800 Subject: [PATCH 53/71] panel cleanup --- experimental/resource/openapi.go | 8 ++++---- experimental/resource/schemabuilder/reflector_test.go | 2 +- experimental/resource/schemabuilder/schema.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/experimental/resource/openapi.go b/experimental/resource/openapi.go index d5ce09f09..c6bc47e45 100644 --- a/experimental/resource/openapi.go +++ b/experimental/resource/openapi.go @@ -14,7 +14,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA return map[string]common.OpenAPIDefinition{ "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.GenericDataQuery": schemaGenericQuery(ref), + "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.DataQuery": schemaDataQuery(ref), "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), } } @@ -55,8 +55,8 @@ func schemaQueryTypeDefinitionSpec(_ common.ReferenceCallback) common.OpenAPIDef } } -func schemaGenericQuery(_ common.ReferenceCallback) common.OpenAPIDefinition { - s, _ := GenericQuerySchema() +func schemaDataQuery(_ common.ReferenceCallback) common.OpenAPIDefinition { + s, _ := DataQuerySchema() if s == nil { s = &spec.Schema{} } @@ -66,7 +66,7 @@ func schemaGenericQuery(_ common.ReferenceCallback) common.OpenAPIDefinition { } // Get the cached feature list (exposed as a k8s resource) -func GenericQuerySchema() (*spec.Schema, error) { +func DataQuerySchema() (*spec.Schema, error) { return loadSchema("query.schema.json") } diff --git a/experimental/resource/schemabuilder/reflector_test.go b/experimental/resource/schemabuilder/reflector_test.go index e7bb74361..d6b4009df 100644 --- a/experimental/resource/schemabuilder/reflector_test.go +++ b/experimental/resource/schemabuilder/reflector_test.go @@ -45,7 +45,7 @@ func TestWriteQuerySchema(t *testing.T) { maybeUpdateFile(t, outfile, query, old) // Make sure the embedded schema is loadable - schema, err := resource.GenericQuerySchema() + schema, err := resource.DataQuerySchema() require.NoError(t, err) require.Equal(t, 8, len(schema.Properties)) diff --git a/experimental/resource/schemabuilder/schema.go b/experimental/resource/schemabuilder/schema.go index 68c62ecd7..18bfaabd5 100644 --- a/experimental/resource/schemabuilder/schema.go +++ b/experimental/resource/schemabuilder/schema.go @@ -38,7 +38,7 @@ type QuerySchemaOptions struct { // Given definitions for a plugin, return a valid spec func GetQuerySchema(opts QuerySchemaOptions) (*spec.Schema, error) { isRequest := opts.Mode == SchemaTypeQueryPayload || opts.Mode == SchemaTypeQueryRequest - generic, err := resource.GenericQuerySchema() + generic, err := resource.DataQuerySchema() if err != nil { return nil, err } From d35e87024634d46326b80d58d63cd0d0ec00377e Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 3 Mar 2024 09:01:03 -0800 Subject: [PATCH 54/71] inline timerange --- experimental/resource/query.go | 47 ++++++++++++++----- experimental/resource/query.schema.json | 6 +-- .../example/query.panel.schema.json | 18 +++---- .../example/query.request.example.json | 6 +-- .../example/query.request.schema.json | 18 +++---- .../resource/schemabuilder/examples.go | 6 ++- 6 files changed, 56 insertions(+), 45 deletions(-) diff --git a/experimental/resource/query.go b/experimental/resource/query.go index 2e862df38..d80746896 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -17,13 +17,8 @@ func init() { //nolint:gochecknoinits } type DataQueryRequest struct { - // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. - // example: now-1h - From string `json:"from,omitempty"` - - // To End time in epoch timestamps in milliseconds or relative using Grafana time units. - // example: now - To string `json:"to,omitempty"` + // Time range applied to each query where it is not specified + TimeRange `json:",inline"` // Each item has a Queries []DataQuery `json:"queries"` @@ -380,10 +375,10 @@ type DataSourceRef struct { // TimeRange represents a time range for a query and is a property of DataQuery. type TimeRange struct { // From is the start time of the query. - From string `json:"from" jsonschema:"example=now-1h"` + From string `json:"from" jsonschema:"example=now-1h,default=now-6h"` // To is the end time of the query. - To string `json:"to" jsonschema:"example=now"` + To string `json:"to" jsonschema:"example=now,default=now"` } // ResultAssertions define the expected response shape and query behavior. This is useful to @@ -397,9 +392,37 @@ type ResultAssertions struct { // contract documentation https://grafana.github.io/dataplane/contract/. TypeVersion data.FrameTypeVersion `json:"typeVersion"` - // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast - MaxBytes int64 `json:"maxBytes,omitempty"` - // Maximum frame count MaxFrames int64 `json:"maxFrames,omitempty"` + + // Once we can support this, adding max bytes would be helpful + // // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast + // MaxBytes int64 `json:"maxBytes,omitempty"` +} + +func (r *ResultAssertions) Validate(frames data.Frames) error { + if r.Type != data.FrameTypeUnknown { + for _, frame := range frames { + if frame.Meta == nil { + return fmt.Errorf("result missing frame type (and metadata)") + } + if frame.Meta.Type == data.FrameTypeUnknown { + // ?? should we try to detect? and see if we can use it as that type? + return fmt.Errorf("expected frame type [%s], but the type is unknown", r.Type) + } + if frame.Meta.Type != r.Type { + return fmt.Errorf("expected frame type [%s], but found [%s]", r.Type, frame.Meta.Type) + } + if !r.TypeVersion.IsZero() { + if r.TypeVersion == frame.Meta.TypeVersion { + return fmt.Errorf("type versions do not match. Expected [%s], but found [%s]", r.TypeVersion, frame.Meta.TypeVersion) + } + } + } + } + + if r.MaxFrames > 0 && len(frames) > int(r.MaxFrames) { + return fmt.Errorf("more than expected frames found") + } + return nil } diff --git a/experimental/resource/query.schema.json b/experimental/resource/query.schema.json index 7c21a8053..8972c3fd1 100644 --- a/experimental/resource/query.schema.json +++ b/experimental/resource/query.schema.json @@ -34,10 +34,6 @@ "minItems": 2, "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." }, - "maxBytes": { - "type": "integer", - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast" - }, "maxFrames": { "type": "integer", "description": "Maximum frame count" @@ -55,6 +51,7 @@ "from": { "type": "string", "description": "From is the start time of the query.", + "default": "now-6h", "examples": [ "now-1h" ] @@ -62,6 +59,7 @@ "to": { "type": "string", "description": "To is the end time of the query.", + "default": "now", "examples": [ "now" ] diff --git a/experimental/resource/schemabuilder/example/query.panel.schema.json b/experimental/resource/schemabuilder/example/query.panel.schema.json index a9c19e5b0..95f65f40e 100644 --- a/experimental/resource/schemabuilder/example/query.panel.schema.json +++ b/experimental/resource/schemabuilder/example/query.panel.schema.json @@ -65,10 +65,6 @@ "typeVersion" ], "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, "maxFrames": { "description": "Maximum frame count", "type": "integer" @@ -114,6 +110,7 @@ "from": { "description": "From is the start time of the query.", "type": "string", + "default": "now-6h", "examples": [ "now-1h" ] @@ -121,6 +118,7 @@ "to": { "description": "To is the end time of the query.", "type": "string", + "default": "now", "examples": [ "now" ] @@ -200,10 +198,6 @@ "typeVersion" ], "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, "maxFrames": { "description": "Maximum frame count", "type": "integer" @@ -275,6 +269,7 @@ "from": { "description": "From is the start time of the query.", "type": "string", + "default": "now-6h", "examples": [ "now-1h" ] @@ -282,6 +277,7 @@ "to": { "description": "To is the end time of the query.", "type": "string", + "default": "now", "examples": [ "now" ] @@ -357,10 +353,6 @@ "typeVersion" ], "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, "maxFrames": { "description": "Maximum frame count", "type": "integer" @@ -406,6 +398,7 @@ "from": { "description": "From is the start time of the query.", "type": "string", + "default": "now-6h", "examples": [ "now-1h" ] @@ -413,6 +406,7 @@ "to": { "description": "To is the end time of the query.", "type": "string", + "default": "now", "examples": [ "now" ] diff --git a/experimental/resource/schemabuilder/example/query.request.example.json b/experimental/resource/schemabuilder/example/query.request.example.json index e48a2f930..4bbf33a43 100644 --- a/experimental/resource/schemabuilder/example/query.request.example.json +++ b/experimental/resource/schemabuilder/example/query.request.example.json @@ -21,11 +21,11 @@ "queryType": "reduce", "maxDataPoints": 1000, "intervalMs": 5, + "expression": "$A", + "reducer": "max", "settings": { "mode": "dropNN" - }, - "expression": "$A", - "reducer": "max" + } } ] } \ No newline at end of file diff --git a/experimental/resource/schemabuilder/example/query.request.schema.json b/experimental/resource/schemabuilder/example/query.request.schema.json index fc9d16478..c88ae04fb 100644 --- a/experimental/resource/schemabuilder/example/query.request.schema.json +++ b/experimental/resource/schemabuilder/example/query.request.schema.json @@ -83,10 +83,6 @@ "typeVersion" ], "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, "maxFrames": { "description": "Maximum frame count", "type": "integer" @@ -132,6 +128,7 @@ "from": { "description": "From is the start time of the query.", "type": "string", + "default": "now-6h", "examples": [ "now-1h" ] @@ -139,6 +136,7 @@ "to": { "description": "To is the end time of the query.", "type": "string", + "default": "now", "examples": [ "now" ] @@ -226,10 +224,6 @@ "typeVersion" ], "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, "maxFrames": { "description": "Maximum frame count", "type": "integer" @@ -301,6 +295,7 @@ "from": { "description": "From is the start time of the query.", "type": "string", + "default": "now-6h", "examples": [ "now-1h" ] @@ -308,6 +303,7 @@ "to": { "description": "To is the end time of the query.", "type": "string", + "default": "now", "examples": [ "now" ] @@ -391,10 +387,6 @@ "typeVersion" ], "properties": { - "maxBytes": { - "description": "Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast", - "type": "integer" - }, "maxFrames": { "description": "Maximum frame count", "type": "integer" @@ -440,6 +432,7 @@ "from": { "description": "From is the start time of the query.", "type": "string", + "default": "now-6h", "examples": [ "now-1h" ] @@ -447,6 +440,7 @@ "to": { "description": "To is the end time of the query.", "type": "string", + "default": "now", "examples": [ "now" ] diff --git a/experimental/resource/schemabuilder/examples.go b/experimental/resource/schemabuilder/examples.go index 8654c50ed..40f4c1d87 100644 --- a/experimental/resource/schemabuilder/examples.go +++ b/experimental/resource/schemabuilder/examples.go @@ -6,8 +6,10 @@ import ( func exampleRequest(defs resource.QueryTypeDefinitionList) resource.DataQueryRequest { rsp := resource.DataQueryRequest{ - From: "now-1h", - To: "now", + TimeRange: resource.TimeRange{ + From: "now-1h", + To: "now", + }, Queries: []resource.DataQuery{}, } From a7986a7fc342cb5d6e12d5b90ce4972a3fbe1bcc Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 3 Mar 2024 11:13:30 -0800 Subject: [PATCH 55/71] more deepcopy --- experimental/resource/query.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/experimental/resource/query.go b/experimental/resource/query.go index d80746896..27479efef 100644 --- a/experimental/resource/query.go +++ b/experimental/resource/query.go @@ -17,13 +17,13 @@ func init() { //nolint:gochecknoinits } type DataQueryRequest struct { - // Time range applied to each query where it is not specified + // Time range applied to each query (when not included in the query body) TimeRange `json:",inline"` - // Each item has a + // Datasource queries Queries []DataQuery `json:"queries"` - // required: false + // Optionally include debug information in the response Debug bool `json:"debug,omitempty"` } @@ -72,6 +72,23 @@ func (g *DataQuery) DeepCopyInto(out *DataQuery) { *out = *clone } +func (g *DataQueryRequest) DeepCopy() *DataQueryRequest { + if g == nil { + return nil + } + out := new(DataQueryRequest) + jj, err := json.Marshal(g) + if err == nil { + _ = json.Unmarshal(jj, out) + } + return out +} + +func (g *DataQueryRequest) DeepCopyInto(out *DataQueryRequest) { + clone := g.DeepCopy() + *out = *clone +} + // Set allows setting values using key/value pairs func (g *DataQuery) Set(key string, val any) *DataQuery { switch key { From 71374e7ef0638beabd6ac4be051d909b8896f8f1 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 3 Mar 2024 19:56:30 -0800 Subject: [PATCH 56/71] hopefully last big package rename --- .../{resource => }/schemabuilder/enums.go | 0 .../schemabuilder/enums_test.go | 4 +- .../schemabuilder/example/query.go | 0 .../example/query.panel.example.json | 0 .../example/query.panel.schema.json | 54 +++---------------- .../example/query.request.example.json | 0 .../example/query.request.schema.json | 54 +++---------------- .../schemabuilder/example/query.types.json | 0 .../schemabuilder/example/query_test.go | 24 ++++----- .../{resource => }/schemabuilder/examples.go | 18 +++---- .../{resource => }/schemabuilder/reflector.go | 52 +++++++++--------- .../schemabuilder/reflector_test.go | 22 ++++---- .../{resource => }/schemabuilder/schema.go | 6 +-- experimental/testdata/folder.golden.txt | 2 +- {experimental/resource => v0alpha1}/metaV1.go | 2 +- .../resource => v0alpha1}/openapi.go | 10 ++-- .../resource => v0alpha1}/openapi_test.go | 2 +- {experimental/resource => v0alpha1}/panel.go | 2 +- .../query.definition.schema.json | 0 {experimental/resource => v0alpha1}/query.go | 6 +-- .../resource => v0alpha1}/query.schema.json | 16 +----- .../resource => v0alpha1}/query_definition.go | 2 +- .../resource => v0alpha1}/query_test.go | 2 +- {experimental/resource => v0alpha1}/schema.go | 2 +- .../resource => v0alpha1}/schema_test.go | 2 +- .../resource => v0alpha1}/settings.go | 2 +- .../testdata/sample_query_results.json | 0 .../resource => v0alpha1}/unstructured.go | 2 +- 28 files changed, 94 insertions(+), 192 deletions(-) rename experimental/{resource => }/schemabuilder/enums.go (100%) rename experimental/{resource => }/schemabuilder/enums_test.go (86%) rename experimental/{resource => }/schemabuilder/example/query.go (100%) rename experimental/{resource => }/schemabuilder/example/query.panel.example.json (100%) rename experimental/{resource => }/schemabuilder/example/query.panel.schema.json (84%) rename experimental/{resource => }/schemabuilder/example/query.request.example.json (100%) rename experimental/{resource => }/schemabuilder/example/query.request.schema.json (86%) rename experimental/{resource => }/schemabuilder/example/query.types.json (100%) rename experimental/{resource => }/schemabuilder/example/query_test.go (65%) rename experimental/{resource => }/schemabuilder/examples.go (62%) rename experimental/{resource => }/schemabuilder/reflector.go (89%) rename experimental/{resource => }/schemabuilder/reflector_test.go (63%) rename experimental/{resource => }/schemabuilder/schema.go (96%) rename {experimental/resource => v0alpha1}/metaV1.go (97%) rename {experimental/resource => v0alpha1}/openapi.go (85%) rename {experimental/resource => v0alpha1}/openapi_test.go (98%) rename {experimental/resource => v0alpha1}/panel.go (95%) rename {experimental/resource => v0alpha1}/query.definition.schema.json (100%) rename {experimental/resource => v0alpha1}/query.go (98%) rename {experimental/resource => v0alpha1}/query.schema.json (83%) rename {experimental/resource => v0alpha1}/query_definition.go (99%) rename {experimental/resource => v0alpha1}/query_test.go (99%) rename {experimental/resource => v0alpha1}/schema.go (98%) rename {experimental/resource => v0alpha1}/schema_test.go (97%) rename {experimental/resource => v0alpha1}/settings.go (98%) rename {experimental/resource => v0alpha1}/testdata/sample_query_results.json (100%) rename {experimental/resource => v0alpha1}/unstructured.go (99%) diff --git a/experimental/resource/schemabuilder/enums.go b/experimental/schemabuilder/enums.go similarity index 100% rename from experimental/resource/schemabuilder/enums.go rename to experimental/schemabuilder/enums.go diff --git a/experimental/resource/schemabuilder/enums_test.go b/experimental/schemabuilder/enums_test.go similarity index 86% rename from experimental/resource/schemabuilder/enums_test.go rename to experimental/schemabuilder/enums_test.go index 4f368dc12..0d0d0b6c3 100644 --- a/experimental/resource/schemabuilder/enums_test.go +++ b/experimental/schemabuilder/enums_test.go @@ -12,7 +12,7 @@ func TestFindEnums(t *testing.T) { t.Run("data", func(t *testing.T) { fields, err := findEnumFields( "github.com/grafana/grafana-plugin-sdk-go/data", - "../../../data") + "../../data") require.NoError(t, err) out, err := json.MarshalIndent(fields, "", " ") @@ -24,7 +24,7 @@ func TestFindEnums(t *testing.T) { t.Run("example", func(t *testing.T) { fields, err := findEnumFields( - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder/example", + "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder/example", "./example") require.NoError(t, err) diff --git a/experimental/resource/schemabuilder/example/query.go b/experimental/schemabuilder/example/query.go similarity index 100% rename from experimental/resource/schemabuilder/example/query.go rename to experimental/schemabuilder/example/query.go diff --git a/experimental/resource/schemabuilder/example/query.panel.example.json b/experimental/schemabuilder/example/query.panel.example.json similarity index 100% rename from experimental/resource/schemabuilder/example/query.panel.example.json rename to experimental/schemabuilder/example/query.panel.example.json diff --git a/experimental/resource/schemabuilder/example/query.panel.schema.json b/experimental/schemabuilder/example/query.panel.schema.json similarity index 84% rename from experimental/resource/schemabuilder/example/query.panel.schema.json rename to experimental/schemabuilder/example/query.panel.schema.json index 95f65f40e..911045841 100644 --- a/experimental/resource/schemabuilder/example/query.panel.schema.json +++ b/experimental/schemabuilder/example/query.panel.schema.json @@ -70,22 +70,8 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", - "type": "string", - "enum": [ - "", - "timeseries-wide", - "timeseries-long", - "timeseries-many", - "timeseries-multi", - "directory-listing", - "table", - "numeric-wide", - "numeric-multi", - "numeric-long", - "log-lines" - ], - "x-enum-description": {} + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -203,22 +189,8 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", - "type": "string", - "enum": [ - "", - "timeseries-wide", - "timeseries-long", - "timeseries-many", - "timeseries-multi", - "directory-listing", - "table", - "numeric-wide", - "numeric-multi", - "numeric-long", - "log-lines" - ], - "x-enum-description": {} + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -358,22 +330,8 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", - "type": "string", - "enum": [ - "", - "timeseries-wide", - "timeseries-long", - "timeseries-many", - "timeseries-multi", - "directory-listing", - "table", - "numeric-wide", - "numeric-multi", - "numeric-long", - "log-lines" - ], - "x-enum-description": {} + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", diff --git a/experimental/resource/schemabuilder/example/query.request.example.json b/experimental/schemabuilder/example/query.request.example.json similarity index 100% rename from experimental/resource/schemabuilder/example/query.request.example.json rename to experimental/schemabuilder/example/query.request.example.json diff --git a/experimental/resource/schemabuilder/example/query.request.schema.json b/experimental/schemabuilder/example/query.request.schema.json similarity index 86% rename from experimental/resource/schemabuilder/example/query.request.schema.json rename to experimental/schemabuilder/example/query.request.schema.json index c88ae04fb..c1e45ed1b 100644 --- a/experimental/resource/schemabuilder/example/query.request.schema.json +++ b/experimental/schemabuilder/example/query.request.schema.json @@ -88,22 +88,8 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", - "type": "string", - "enum": [ - "", - "timeseries-wide", - "timeseries-long", - "timeseries-many", - "timeseries-multi", - "directory-listing", - "table", - "numeric-wide", - "numeric-multi", - "numeric-long", - "log-lines" - ], - "x-enum-description": {} + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -229,22 +215,8 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", - "type": "string", - "enum": [ - "", - "timeseries-wide", - "timeseries-long", - "timeseries-many", - "timeseries-multi", - "directory-listing", - "table", - "numeric-wide", - "numeric-multi", - "numeric-long", - "log-lines" - ], - "x-enum-description": {} + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -392,22 +364,8 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", - "type": "string", - "enum": [ - "", - "timeseries-wide", - "timeseries-long", - "timeseries-many", - "timeseries-multi", - "directory-listing", - "table", - "numeric-wide", - "numeric-multi", - "numeric-long", - "log-lines" - ], - "x-enum-description": {} + "description": "Type asserts that the frame matches a known type structure.", + "type": "string" }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", diff --git a/experimental/resource/schemabuilder/example/query.types.json b/experimental/schemabuilder/example/query.types.json similarity index 100% rename from experimental/resource/schemabuilder/example/query.types.json rename to experimental/schemabuilder/example/query.types.json diff --git a/experimental/resource/schemabuilder/example/query_test.go b/experimental/schemabuilder/example/query_test.go similarity index 65% rename from experimental/resource/schemabuilder/example/query_test.go rename to experimental/schemabuilder/example/query_test.go index 0a9e484ca..7f5fea12c 100644 --- a/experimental/resource/schemabuilder/example/query_test.go +++ b/experimental/schemabuilder/example/query_test.go @@ -4,8 +4,8 @@ import ( "reflect" "testing" - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder" + "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" "github.com/stretchr/testify/require" ) @@ -13,7 +13,7 @@ func TestQueryTypeDefinitions(t *testing.T) { builder, err := schemabuilder.NewSchemaBuilder(schemabuilder.BuilderOptions{ PluginID: []string{"__expr__"}, ScanCode: []schemabuilder.CodePaths{{ - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/schemabuilder/example", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder/example", CodePath: "./", }}, Enums: []reflect.Type{ @@ -23,30 +23,30 @@ func TestQueryTypeDefinitions(t *testing.T) { }) require.NoError(t, err) err = builder.AddQueries(schemabuilder.QueryTypeInfo{ - Discriminators: resource.NewDiscriminators("queryType", QueryTypeMath), + Discriminators: sdkapi.NewDiscriminators("queryType", QueryTypeMath), GoType: reflect.TypeOf(&MathQuery{}), - Examples: []resource.QueryExample{ + Examples: []sdkapi.QueryExample{ { Name: "constant addition", - SaveModel: resource.AsUnstructured(MathQuery{ + SaveModel: sdkapi.AsUnstructured(MathQuery{ Expression: "$A + 11", }), }, { Name: "math with two queries", - SaveModel: resource.AsUnstructured(MathQuery{ + SaveModel: sdkapi.AsUnstructured(MathQuery{ Expression: "$A - $B", }), }, }, }, schemabuilder.QueryTypeInfo{ - Discriminators: resource.NewDiscriminators("queryType", QueryTypeReduce), + Discriminators: sdkapi.NewDiscriminators("queryType", QueryTypeReduce), GoType: reflect.TypeOf(&ReduceQuery{}), - Examples: []resource.QueryExample{ + Examples: []sdkapi.QueryExample{ { Name: "get max value", - SaveModel: resource.AsUnstructured(ReduceQuery{ + SaveModel: sdkapi.AsUnstructured(ReduceQuery{ Expression: "$A", Reducer: ReducerMax, Settings: ReduceSettings{ @@ -57,9 +57,9 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }, schemabuilder.QueryTypeInfo{ - Discriminators: resource.NewDiscriminators("queryType", QueryTypeResample), + Discriminators: sdkapi.NewDiscriminators("queryType", QueryTypeResample), GoType: reflect.TypeOf(&ResampleQuery{}), - Examples: []resource.QueryExample{}, + Examples: []sdkapi.QueryExample{}, }) require.NoError(t, err) diff --git a/experimental/resource/schemabuilder/examples.go b/experimental/schemabuilder/examples.go similarity index 62% rename from experimental/resource/schemabuilder/examples.go rename to experimental/schemabuilder/examples.go index 40f4c1d87..7880bcc7f 100644 --- a/experimental/resource/schemabuilder/examples.go +++ b/experimental/schemabuilder/examples.go @@ -1,22 +1,22 @@ package schemabuilder import ( - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" ) -func exampleRequest(defs resource.QueryTypeDefinitionList) resource.DataQueryRequest { - rsp := resource.DataQueryRequest{ - TimeRange: resource.TimeRange{ +func exampleRequest(defs sdkapi.QueryTypeDefinitionList) sdkapi.DataQueryRequest { + rsp := sdkapi.DataQueryRequest{ + TimeRange: sdkapi.TimeRange{ From: "now-1h", To: "now", }, - Queries: []resource.DataQuery{}, + Queries: []sdkapi.DataQuery{}, } for _, def := range defs.Items { for _, sample := range def.Spec.Examples { if sample.SaveModel.Object != nil { - q := resource.NewDataQuery(sample.SaveModel.Object) + q := sdkapi.NewDataQuery(sample.SaveModel.Object) q.RefID = string(rune('A' + len(rsp.Queries))) for _, dis := range def.Spec.Discriminators { _ = q.Set(dis.Field, dis.Value) @@ -36,13 +36,13 @@ func exampleRequest(defs resource.QueryTypeDefinitionList) resource.DataQueryReq return rsp } -func examplePanelTargets(ds *resource.DataSourceRef, defs resource.QueryTypeDefinitionList) []resource.DataQuery { - targets := []resource.DataQuery{} +func examplePanelTargets(ds *sdkapi.DataSourceRef, defs sdkapi.QueryTypeDefinitionList) []sdkapi.DataQuery { + targets := []sdkapi.DataQuery{} for _, def := range defs.Items { for _, sample := range def.Spec.Examples { if sample.SaveModel.Object != nil { - q := resource.NewDataQuery(sample.SaveModel.Object) + q := sdkapi.NewDataQuery(sample.SaveModel.Object) q.Datasource = ds q.RefID = string(rune('A' + len(targets))) for _, dis := range def.Spec.Discriminators { diff --git a/experimental/resource/schemabuilder/reflector.go b/experimental/schemabuilder/reflector.go similarity index 89% rename from experimental/resource/schemabuilder/reflector.go rename to experimental/schemabuilder/reflector.go index 91db27556..32f0fce1f 100644 --- a/experimental/resource/schemabuilder/reflector.go +++ b/experimental/schemabuilder/reflector.go @@ -12,7 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" "github.com/invopop/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -27,8 +27,8 @@ import ( type Builder struct { opts BuilderOptions reflector *jsonschema.Reflector // Needed to use comments - query []resource.QueryTypeDefinition - setting []resource.SettingsDefinition + query []sdkapi.QueryTypeDefinition + setting []sdkapi.SettingsDefinition } type CodePaths struct { @@ -56,18 +56,18 @@ type QueryTypeInfo struct { // Optional description Description string // Optional discriminators - Discriminators []resource.DiscriminatorFieldValue + Discriminators []sdkapi.DiscriminatorFieldValue // Raw GO type used for reflection GoType reflect.Type // Add sample queries - Examples []resource.QueryExample + Examples []sdkapi.QueryExample } type SettingTypeInfo struct { // The management name Name string // Optional discriminators - Discriminators []resource.DiscriminatorFieldValue + Discriminators []sdkapi.DiscriminatorFieldValue // Raw GO type used for reflection GoType reflect.Type // Map[string]string @@ -94,11 +94,11 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { }, AdditionalProperties: jsonschema.TrueSchema, }, - reflect.TypeOf(resource.Unstructured{}): { + reflect.TypeOf(sdkapi.Unstructured{}): { Type: "object", AdditionalProperties: jsonschema.TrueSchema, }, - reflect.TypeOf(resource.JSONSchema{}): { + reflect.TypeOf(sdkapi.JSONSchema{}): { Type: "object", Ref: draft04, }, @@ -176,14 +176,14 @@ func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { return err } - b.query = append(b.query, resource.QueryTypeDefinition{ - ObjectMeta: resource.ObjectMeta{ + b.query = append(b.query, sdkapi.QueryTypeDefinition{ + ObjectMeta: sdkapi.ObjectMeta{ Name: name, }, - Spec: resource.QueryTypeDefinitionSpec{ + Spec: sdkapi.QueryTypeDefinitionSpec{ Description: info.Description, Discriminators: info.Discriminators, - Schema: resource.JSONSchema{ + Schema: sdkapi.JSONSchema{ Spec: spec, }, Examples: info.Examples, @@ -216,13 +216,13 @@ func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { return err } - b.setting = append(b.setting, resource.SettingsDefinition{ - ObjectMeta: resource.ObjectMeta{ + b.setting = append(b.setting, sdkapi.SettingsDefinition{ + ObjectMeta: sdkapi.ObjectMeta{ Name: name, }, - Spec: resource.SettingsDefinitionSpec{ + Spec: sdkapi.SettingsDefinitionSpec{ Discriminators: info.Discriminators, - JSONDataSchema: resource.JSONSchema{ + JSONDataSchema: sdkapi.JSONSchema{ Spec: spec, }, }, @@ -235,15 +235,15 @@ func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { // When placed in `static/schema/query.types.json` folder of a plugin distribution, // it can be used to advertise various query types // If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.QueryTypeDefinitionList { +func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) sdkapi.QueryTypeDefinitionList { t.Helper() outfile := filepath.Join(outdir, "query.types.json") now := time.Now().UTC() rv := fmt.Sprintf("%d", now.UnixMilli()) - defs := resource.QueryTypeDefinitionList{} - byName := make(map[string]*resource.QueryTypeDefinition) + defs := sdkapi.QueryTypeDefinitionList{} + byName := make(map[string]*sdkapi.QueryTypeDefinition) body, err := os.ReadFile(outfile) if err == nil { err = json.Unmarshal(body, &defs) @@ -266,8 +266,8 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu defs.Items = append(defs.Items, def) } else { - x := resource.AsUnstructured(def.Spec) - y := resource.AsUnstructured(found.Spec) + x := sdkapi.AsUnstructured(def.Spec) + y := sdkapi.AsUnstructured(found.Spec) if diff := cmp.Diff(stripNilValues(x.Object), stripNilValues(y.Object), cmpopts.EquateEmpty()); diff != "" { fmt.Printf("Spec changed:\n%s\n", diff) found.ObjectMeta.ResourceVersion = rv @@ -299,10 +299,10 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, schema, body) - panel := resource.PseudoPanel{ + panel := sdkapi.PseudoPanel{ Type: "table", } - panel.Targets = examplePanelTargets(&resource.DataSourceRef{ + panel.Targets = examplePanelTargets(&sdkapi.DataSourceRef{ Type: b.opts.PluginID[0], UID: "TheUID", }, defs) @@ -352,14 +352,14 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) resource.Qu // When placed in `static/schema/query.schema.json` folder of a plugin distribution, // it can be used to advertise various query types // If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) resource.SettingsDefinitionList { +func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) sdkapi.SettingsDefinitionList { t.Helper() now := time.Now().UTC() rv := fmt.Sprintf("%d", now.UnixMilli()) - defs := resource.SettingsDefinitionList{} - byName := make(map[string]*resource.SettingsDefinition) + defs := sdkapi.SettingsDefinitionList{} + byName := make(map[string]*sdkapi.SettingsDefinition) body, err := os.ReadFile(outfile) if err == nil { err = json.Unmarshal(body, &defs) diff --git a/experimental/resource/schemabuilder/reflector_test.go b/experimental/schemabuilder/reflector_test.go similarity index 63% rename from experimental/resource/schemabuilder/reflector_test.go rename to experimental/schemabuilder/reflector_test.go index d6b4009df..76cadb770 100644 --- a/experimental/resource/schemabuilder/reflector_test.go +++ b/experimental/schemabuilder/reflector_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" "github.com/invopop/jsonschema" "github.com/stretchr/testify/require" ) @@ -16,12 +16,12 @@ func TestWriteQuerySchema(t *testing.T) { PluginID: []string{"dummy"}, ScanCode: []CodePaths{ { - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/resource/dotdothack", - CodePath: "../", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/v0alpha1/dummy", + CodePath: "../../v0alpha1", }, { - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/data", - CodePath: "../../../data", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/data/dummy", + CodePath: "../../data", }, }, Enums: []reflect.Type{ @@ -30,7 +30,7 @@ func TestWriteQuerySchema(t *testing.T) { }) require.NoError(t, err) - query := builder.reflector.Reflect(&resource.CommonQueryProperties{}) + query := builder.reflector.Reflect(&sdkapi.CommonQueryProperties{}) updateEnumDescriptions(query) query.ID = "" query.Version = draft04 // used by kube-openapi @@ -40,24 +40,24 @@ func TestWriteQuerySchema(t *testing.T) { // // Hide this old property query.Properties.Delete("datasourceId") - outfile := "../query.schema.json" + outfile := "../../v0alpha1/query.schema.json" old, _ := os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) // Make sure the embedded schema is loadable - schema, err := resource.DataQuerySchema() + schema, err := sdkapi.DataQuerySchema() require.NoError(t, err) require.Equal(t, 8, len(schema.Properties)) // Add schema for query type definition - query = builder.reflector.Reflect(&resource.QueryTypeDefinitionSpec{}) + query = builder.reflector.Reflect(&sdkapi.QueryTypeDefinitionSpec{}) updateEnumDescriptions(query) query.ID = "" query.Version = draft04 // used by kube-openapi - outfile = "../query.definition.schema.json" + outfile = "../../v0alpha1/query.definition.schema.json" old, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) - def := resource.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/experimental/resource.QueryTypeDefinitionSpec"] + def := sdkapi.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/v0alpha1.QueryTypeDefinitionSpec"] require.Equal(t, query.Properties.Len(), len(def.Schema.Properties)) } diff --git a/experimental/resource/schemabuilder/schema.go b/experimental/schemabuilder/schema.go similarity index 96% rename from experimental/resource/schemabuilder/schema.go rename to experimental/schemabuilder/schema.go index 18bfaabd5..1113cd56a 100644 --- a/experimental/resource/schemabuilder/schema.go +++ b/experimental/schemabuilder/schema.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" "k8s.io/kube-openapi/pkg/validation/spec" ) @@ -31,14 +31,14 @@ const ( type QuerySchemaOptions struct { PluginID []string - QueryTypes []resource.QueryTypeDefinition + QueryTypes []sdkapi.QueryTypeDefinition Mode SchemaType } // Given definitions for a plugin, return a valid spec func GetQuerySchema(opts QuerySchemaOptions) (*spec.Schema, error) { isRequest := opts.Mode == SchemaTypeQueryPayload || opts.Mode == SchemaTypeQueryRequest - generic, err := resource.DataQuerySchema() + generic, err := sdkapi.DataQuerySchema() if err != nil { return nil, err } diff --git a/experimental/testdata/folder.golden.txt b/experimental/testdata/folder.golden.txt index 7abe047c3..767f5a01d 100644 --- a/experimental/testdata/folder.golden.txt +++ b/experimental/testdata/folder.golden.txt @@ -29,4 +29,4 @@ Dimensions: 2 Fields by 20 Rows ====== TEST DATA RESPONSE (arrow base64) ====== -FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAKAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAAAQEAAAAAAABgAQAAAAAAAAAAAAAAAAAAYAEAAAAAAABUAAAAAAAAALgBAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA6wAAAPkAAAABAQAAAAAAAFJFQURNRS5tZGFjdGlvbnNhdXRoY2xpZW50ZGF0YXNvdXJjZXRlc3RlMmVlcnJvcnNvdXJjZWZlYXR1cmV0b2dnbGVzZmlsZWluZm8uZ29maWxlaW5mb190ZXN0LmdvZnJhbWVfc29ydGVyLmdvZnJhbWVfc29ydGVyX3Rlc3QuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlci5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyX3Rlc3QuZ29odHRwX2xvZ2dlcm1hY3Jvc21vY2tvYXV0aHRva2VucmV0cmlldmVycmVzb3VyY2VyZXN0X2NsaWVudC5nb3Rlc3RkYXRhAAAAAAAAAAAAAAAAAAAACQAAABIAAAAbAAAAJAAAAC0AAAA2AAAANgAAADYAAAA2AAAANgAAADYAAAA2AAAAPwAAAEgAAABRAAAAWgAAAGMAAABjAAAAbAAAAAAAAABkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnkAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAANgBAAAAAAAA4AAAAAAAAAAoAgAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAuAAAAAMAAABMAAAAKAAAAAQAAADA/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAOD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAP///wgAAABQAAAARAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwidHlwZVZlcnNpb24iOlswLDBdLCJwYXRoU2VwYXJhdG9yIjoiLyJ9AAAAAAQAAABtZXRhAAAAAAIAAAB4AAAABAAAAKL///8UAAAAPAAAADwAAAAAAAAFOAAAAAEAAAAEAAAAkP///wgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA+AEAAEFSUk9XMQ== +FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAKAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAABgEAAAAAAABgAQAAAAAAAAAAAAAAAAAAYAEAAAAAAABUAAAAAAAAALgBAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA8QAAAP4AAAAGAQAAAAAAAFJFQURNRS5tZGFjdGlvbnNhdXRoY2xpZW50ZGF0YXNvdXJjZXRlc3RlMmVlcnJvcnNvdXJjZWZlYXR1cmV0b2dnbGVzZmlsZWluZm8uZ29maWxlaW5mb190ZXN0LmdvZnJhbWVfc29ydGVyLmdvZnJhbWVfc29ydGVyX3Rlc3QuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlci5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyX3Rlc3QuZ29odHRwX2xvZ2dlcm1hY3Jvc21vY2tvYXV0aHRva2VucmV0cmlldmVycmVzdF9jbGllbnQuZ29zY2hlbWFidWlsZGVydGVzdGRhdGEAAAAAAAAAAAAACQAAABIAAAAbAAAAJAAAAC0AAAA2AAAANgAAADYAAAA2AAAANgAAADYAAAA2AAAAPwAAAEgAAABRAAAAWgAAAFoAAABjAAAAbAAAAAAAAABkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnkAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAANgBAAAAAAAA4AAAAAAAAAAoAgAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAuAAAAAMAAABMAAAAKAAAAAQAAADA/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAOD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAP///wgAAABQAAAARAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwidHlwZVZlcnNpb24iOlswLDBdLCJwYXRoU2VwYXJhdG9yIjoiLyJ9AAAAAAQAAABtZXRhAAAAAAIAAAB4AAAABAAAAKL///8UAAAAPAAAADwAAAAAAAAFOAAAAAEAAAAEAAAAkP///wgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA+AEAAEFSUk9XMQ== diff --git a/experimental/resource/metaV1.go b/v0alpha1/metaV1.go similarity index 97% rename from experimental/resource/metaV1.go rename to v0alpha1/metaV1.go index 47fbee164..042f01629 100644 --- a/experimental/resource/metaV1.go +++ b/v0alpha1/metaV1.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 // ObjectMeta is a struct that aims to "look" like a real kubernetes object when // written to JSON, however it does not require the pile of dependencies diff --git a/experimental/resource/openapi.go b/v0alpha1/openapi.go similarity index 85% rename from experimental/resource/openapi.go rename to v0alpha1/openapi.go index c6bc47e45..2b22851bf 100644 --- a/experimental/resource/openapi.go +++ b/v0alpha1/openapi.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 import ( "embed" @@ -12,10 +12,10 @@ var f embed.FS func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ - "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), - "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.DataQuery": schemaDataQuery(ref), - "github.com/grafana/grafana-plugin-sdk-go/experimental/resource.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), + "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), + "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), + "github.com/grafana/grafana-plugin-sdk-go/v0alpha1.DataQuery": schemaDataQuery(ref), + "github.com/grafana/grafana-plugin-sdk-go/v0alpha1.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), } } diff --git a/experimental/resource/openapi_test.go b/v0alpha1/openapi_test.go similarity index 98% rename from experimental/resource/openapi_test.go rename to v0alpha1/openapi_test.go index d3dfe12af..06c9eab88 100644 --- a/experimental/resource/openapi_test.go +++ b/v0alpha1/openapi_test.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 import ( "encoding/json" diff --git a/experimental/resource/panel.go b/v0alpha1/panel.go similarity index 95% rename from experimental/resource/panel.go rename to v0alpha1/panel.go index 252f594f5..cb2f62e39 100644 --- a/experimental/resource/panel.go +++ b/v0alpha1/panel.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 type PseudoPanel struct { // Numeric panel id diff --git a/experimental/resource/query.definition.schema.json b/v0alpha1/query.definition.schema.json similarity index 100% rename from experimental/resource/query.definition.schema.json rename to v0alpha1/query.definition.schema.json diff --git a/experimental/resource/query.go b/v0alpha1/query.go similarity index 98% rename from experimental/resource/query.go rename to v0alpha1/query.go index 27479efef..e82014096 100644 --- a/experimental/resource/query.go +++ b/v0alpha1/query.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 import ( "encoding/json" @@ -12,8 +12,8 @@ import ( ) func init() { //nolint:gochecknoinits - jsoniter.RegisterTypeEncoder("resource.DataQuery", &genericQueryCodec{}) - jsoniter.RegisterTypeDecoder("resource.DataQuery", &genericQueryCodec{}) + jsoniter.RegisterTypeEncoder("v0alpha1.DataQuery", &genericQueryCodec{}) + jsoniter.RegisterTypeDecoder("v0alpha1.DataQuery", &genericQueryCodec{}) } type DataQueryRequest struct { diff --git a/experimental/resource/query.schema.json b/v0alpha1/query.schema.json similarity index 83% rename from experimental/resource/query.schema.json rename to v0alpha1/query.schema.json index 8972c3fd1..15634db5b 100644 --- a/experimental/resource/query.schema.json +++ b/v0alpha1/query.schema.json @@ -9,21 +9,7 @@ "properties": { "type": { "type": "string", - "enum": [ - "", - "timeseries-wide", - "timeseries-long", - "timeseries-many", - "timeseries-multi", - "directory-listing", - "table", - "numeric-wide", - "numeric-multi", - "numeric-long", - "log-lines" - ], - "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", - "x-enum-description": {} + "description": "Type asserts that the frame matches a known type structure." }, "typeVersion": { "items": { diff --git a/experimental/resource/query_definition.go b/v0alpha1/query_definition.go similarity index 99% rename from experimental/resource/query_definition.go rename to v0alpha1/query_definition.go index 32203efc2..e5e977bac 100644 --- a/experimental/resource/query_definition.go +++ b/v0alpha1/query_definition.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 import ( "encoding/json" diff --git a/experimental/resource/query_test.go b/v0alpha1/query_test.go similarity index 99% rename from experimental/resource/query_test.go rename to v0alpha1/query_test.go index 41877fb61..c084ead40 100644 --- a/experimental/resource/query_test.go +++ b/v0alpha1/query_test.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 import ( "encoding/json" diff --git a/experimental/resource/schema.go b/v0alpha1/schema.go similarity index 98% rename from experimental/resource/schema.go rename to v0alpha1/schema.go index 85d7b2fdd..a834b3364 100644 --- a/experimental/resource/schema.go +++ b/v0alpha1/schema.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 import ( "encoding/json" diff --git a/experimental/resource/schema_test.go b/v0alpha1/schema_test.go similarity index 97% rename from experimental/resource/schema_test.go rename to v0alpha1/schema_test.go index 061ac12d7..ced3e0437 100644 --- a/experimental/resource/schema_test.go +++ b/v0alpha1/schema_test.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 import ( "encoding/json" diff --git a/experimental/resource/settings.go b/v0alpha1/settings.go similarity index 98% rename from experimental/resource/settings.go rename to v0alpha1/settings.go index 68f86a2ce..6fdfdf83f 100644 --- a/experimental/resource/settings.go +++ b/v0alpha1/settings.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 // SettingsDefinition is a kubernetes shaped object that represents a single query definition type SettingsDefinition struct { diff --git a/experimental/resource/testdata/sample_query_results.json b/v0alpha1/testdata/sample_query_results.json similarity index 100% rename from experimental/resource/testdata/sample_query_results.json rename to v0alpha1/testdata/sample_query_results.json diff --git a/experimental/resource/unstructured.go b/v0alpha1/unstructured.go similarity index 99% rename from experimental/resource/unstructured.go rename to v0alpha1/unstructured.go index 625fdd848..dbf87430c 100644 --- a/experimental/resource/unstructured.go +++ b/v0alpha1/unstructured.go @@ -1,4 +1,4 @@ -package resource +package v0alpha1 import ( "encoding/json" From d2c2e90aa9d2b9f108a2becdbd53f21485943251 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 3 Mar 2024 20:38:43 -0800 Subject: [PATCH 57/71] loading frame types --- experimental/schemabuilder/reflector.go | 14 ++++++++++++-- experimental/schemabuilder/reflector_test.go | 4 ++-- v0alpha1/query.schema.json | 16 +++++++++++++++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/experimental/schemabuilder/reflector.go b/experimental/schemabuilder/reflector.go index 32f0fce1f..246152a8a 100644 --- a/experimental/schemabuilder/reflector.go +++ b/experimental/schemabuilder/reflector.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" "time" @@ -82,7 +83,14 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { r := new(jsonschema.Reflector) r.DoNotReference = true for _, scan := range opts.ScanCode { - if err := r.AddGoComments(scan.BasePackage, scan.CodePath); err != nil { + base := scan.BasePackage + for _, v := range strings.Split(scan.CodePath, "/") { + if v == "." || v == ".." { + continue + } + base += "/dummy" // fixes the resolution + } + if err := r.AddGoComments(base, scan.CodePath); err != nil { return nil, err } } @@ -118,8 +126,10 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { } for _, etype := range opts.Enums { + name := etype.Name() + pack := etype.PkgPath() for _, f := range fields { - if f.Name == etype.Name() && f.Package == etype.PkgPath() { + if f.Name == name && f.Package == pack { enumValueDescriptions := map[string]string{} s := &jsonschema.Schema{ Type: "string", diff --git a/experimental/schemabuilder/reflector_test.go b/experimental/schemabuilder/reflector_test.go index 76cadb770..a936300f4 100644 --- a/experimental/schemabuilder/reflector_test.go +++ b/experimental/schemabuilder/reflector_test.go @@ -16,11 +16,11 @@ func TestWriteQuerySchema(t *testing.T) { PluginID: []string{"dummy"}, ScanCode: []CodePaths{ { - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/v0alpha1/dummy", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/v0alpha1", CodePath: "../../v0alpha1", }, { - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/data/dummy", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/data", CodePath: "../../data", }, }, diff --git a/v0alpha1/query.schema.json b/v0alpha1/query.schema.json index 15634db5b..8972c3fd1 100644 --- a/v0alpha1/query.schema.json +++ b/v0alpha1/query.schema.json @@ -9,7 +9,21 @@ "properties": { "type": { "type": "string", - "description": "Type asserts that the frame matches a known type structure." + "enum": [ + "", + "timeseries-wide", + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "x-enum-description": {} }, "typeVersion": { "items": { From 0e568d62d6fe9b21ebea8d3b214b9436589867aa Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 3 Mar 2024 20:45:24 -0800 Subject: [PATCH 58/71] loading frame types --- experimental/schemabuilder/reflector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experimental/schemabuilder/reflector.go b/experimental/schemabuilder/reflector.go index 246152a8a..435d0995d 100644 --- a/experimental/schemabuilder/reflector.go +++ b/experimental/schemabuilder/reflector.go @@ -85,7 +85,7 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { for _, scan := range opts.ScanCode { base := scan.BasePackage for _, v := range strings.Split(scan.CodePath, "/") { - if v == "." || v == ".." { + if v == "" || v == "." || v == ".." { continue } base += "/dummy" // fixes the resolution From e7e1b68a000ee3ce1620be88c7b44789b8b82479 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 3 Mar 2024 21:12:34 -0800 Subject: [PATCH 59/71] fix test --- .../example/query.panel.schema.json | 54 ++++++++++++++++--- .../example/query.request.schema.json | 54 ++++++++++++++++--- 2 files changed, 96 insertions(+), 12 deletions(-) diff --git a/experimental/schemabuilder/example/query.panel.schema.json b/experimental/schemabuilder/example/query.panel.schema.json index 911045841..95f65f40e 100644 --- a/experimental/schemabuilder/example/query.panel.schema.json +++ b/experimental/schemabuilder/example/query.panel.schema.json @@ -70,8 +70,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "type": "string", + "enum": [ + "", + "timeseries-wide", + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -189,8 +203,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "type": "string", + "enum": [ + "", + "timeseries-wide", + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -330,8 +358,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "type": "string", + "enum": [ + "", + "timeseries-wide", + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", diff --git a/experimental/schemabuilder/example/query.request.schema.json b/experimental/schemabuilder/example/query.request.schema.json index c1e45ed1b..c88ae04fb 100644 --- a/experimental/schemabuilder/example/query.request.schema.json +++ b/experimental/schemabuilder/example/query.request.schema.json @@ -88,8 +88,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "type": "string", + "enum": [ + "", + "timeseries-wide", + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -215,8 +229,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "type": "string", + "enum": [ + "", + "timeseries-wide", + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", @@ -364,8 +392,22 @@ "type": "integer" }, "type": { - "description": "Type asserts that the frame matches a known type structure.", - "type": "string" + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "type": "string", + "enum": [ + "", + "timeseries-wide", + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "x-enum-description": {} }, "typeVersion": { "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/.", From 81d7990484ccbb2e2add1788ad567f1f74534265 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Sun, 3 Mar 2024 21:38:45 -0800 Subject: [PATCH 60/71] remove unused settings configs --- experimental/schemabuilder/reflector.go | 96 ------------------------- v0alpha1/settings.go | 40 ----------- 2 files changed, 136 deletions(-) delete mode 100644 v0alpha1/settings.go diff --git a/experimental/schemabuilder/reflector.go b/experimental/schemabuilder/reflector.go index 435d0995d..1b8d427ef 100644 --- a/experimental/schemabuilder/reflector.go +++ b/experimental/schemabuilder/reflector.go @@ -29,7 +29,6 @@ type Builder struct { opts BuilderOptions reflector *jsonschema.Reflector // Needed to use comments query []sdkapi.QueryTypeDefinition - setting []sdkapi.SettingsDefinition } type CodePaths struct { @@ -203,44 +202,6 @@ func (b *Builder) AddQueries(inputs ...QueryTypeInfo) error { return nil } -func (b *Builder) AddSettings(inputs ...SettingTypeInfo) error { - for _, info := range inputs { - name := info.Name - if name == "" { - return fmt.Errorf("missing name") - } - - schema := b.reflector.ReflectFromType(info.GoType) - if schema == nil { - return fmt.Errorf("missing schema") - } - - updateEnumDescriptions(schema) - - // used by kube-openapi - schema.Version = draft04 - schema.ID = "" - schema.Anchor = "" - spec, err := asJSONSchema(schema) - if err != nil { - return err - } - - b.setting = append(b.setting, sdkapi.SettingsDefinition{ - ObjectMeta: sdkapi.ObjectMeta{ - Name: name, - }, - Spec: sdkapi.SettingsDefinitionSpec{ - Discriminators: info.Discriminators, - JSONDataSchema: sdkapi.JSONSchema{ - Spec: spec, - }, - }, - }) - } - return nil -} - // Update the schema definition file // When placed in `static/schema/query.types.json` folder of a plugin distribution, // it can be used to advertise various query types @@ -358,63 +319,6 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) sdkapi.Quer return defs } -// Update the schema definition file -// When placed in `static/schema/query.schema.json` folder of a plugin distribution, -// it can be used to advertise various query types -// If the spec contents have changed, the test will fail (but still update the output) -func (b *Builder) UpdateSettingsDefinition(t *testing.T, outfile string) sdkapi.SettingsDefinitionList { - t.Helper() - - now := time.Now().UTC() - rv := fmt.Sprintf("%d", now.UnixMilli()) - - defs := sdkapi.SettingsDefinitionList{} - byName := make(map[string]*sdkapi.SettingsDefinition) - body, err := os.ReadFile(outfile) - if err == nil { - err = json.Unmarshal(body, &defs) - if err == nil { - for i, def := range defs.Items { - byName[def.ObjectMeta.Name] = &defs.Items[i] - } - } - } - defs.Kind = "SettingsDefinitionList" - defs.APIVersion = "common.grafana.app/v0alpha1" - - // The updated schemas - for _, def := range b.setting { - found, ok := byName[def.ObjectMeta.Name] - if !ok { - defs.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.ResourceVersion = rv - def.ObjectMeta.CreationTimestamp = now.Format(time.RFC3339) - - defs.Items = append(defs.Items, def) - } else { - var o1, o2 interface{} - b1, _ := json.Marshal(def.Spec) - b2, _ := json.Marshal(found.Spec) - _ = json.Unmarshal(b1, &o1) - _ = json.Unmarshal(b2, &o2) - if !reflect.DeepEqual(o1, o2) { - found.ObjectMeta.ResourceVersion = rv - found.Spec = def.Spec - } - delete(byName, def.ObjectMeta.Name) - } - } - - if defs.ObjectMeta.ResourceVersion == "" { - defs.ObjectMeta.ResourceVersion = rv - } - - if len(byName) > 0 { - require.FailNow(t, "settings type removed, manually update (for now)") - } - return defs -} - func maybeUpdateFile(t *testing.T, outfile string, value any, body []byte) { t.Helper() diff --git a/v0alpha1/settings.go b/v0alpha1/settings.go deleted file mode 100644 index 6fdfdf83f..000000000 --- a/v0alpha1/settings.go +++ /dev/null @@ -1,40 +0,0 @@ -package v0alpha1 - -// SettingsDefinition is a kubernetes shaped object that represents a single query definition -type SettingsDefinition struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` - - Spec SettingsDefinitionSpec `json:"spec,omitempty"` -} - -// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types -// For simple data sources, there may be only a single query type, however when multiple types -// exist they must be clearly specified with distinct discriminator field+value pairs -type SettingsDefinitionList struct { - TypeMeta `json:",inline"` - ObjectMeta `json:"metadata,omitempty"` - - Items []SettingsDefinition `json:"items"` -} - -type SettingsDefinitionSpec struct { - // Multiple schemas can be defined using discriminators - Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` - - // Describe whe the query type is for - Description string `json:"description,omitempty"` - - // The query schema represents the properties that can be sent to the API - // In many cases, this may be the same properties that are saved in a dashboard - // In the case where the save model is different, we must also specify a save model - JSONDataSchema JSONSchema `json:"jsonDataSchema"` - - // JSON schema defining the properties needed in secure json - // NOTE all properties must be string values! - SecureProperties JSONSchema `json:"secureJsonSchema"` - - // Changelog defines the changed from the previous version - // All changes in the same version *must* be backwards compatible - // Only notable changes will be shown here, for the full version history see git! - Changelog []string `json:"changelog,omitempty"` -} From 0fe8c9cd6e41238f25ed28eb7503d795213173e7 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 4 Mar 2024 09:02:48 -0800 Subject: [PATCH 61/71] move panel to schema builder --- experimental/schemabuilder/panel.go | 22 ++++++++++++++++++++++ experimental/schemabuilder/reflector.go | 2 +- v0alpha1/panel.go | 18 ------------------ v0alpha1/query.go | 3 +++ 4 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 experimental/schemabuilder/panel.go delete mode 100644 v0alpha1/panel.go diff --git a/experimental/schemabuilder/panel.go b/experimental/schemabuilder/panel.go new file mode 100644 index 000000000..7d47f81bf --- /dev/null +++ b/experimental/schemabuilder/panel.go @@ -0,0 +1,22 @@ +package schemabuilder + +import "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" + +// This is only used to write out a sample panel query +// It is not public and not intended to represent a real panel +type pseudoPanel struct { + // Numeric panel id + ID int `json:"id,omitempty"` + + // The panel plugin type + Type string `json:"type"` + + // The panel title + Title string `json:"title,omitempty"` + + // This should no longer be necessary since each target has the datasource reference + Datasource *v0alpha1.DataSourceRef `json:"datasource,omitempty"` + + // The query targets + Targets []v0alpha1.DataQuery `json:"targets"` +} diff --git a/experimental/schemabuilder/reflector.go b/experimental/schemabuilder/reflector.go index 1b8d427ef..c5a9a0f20 100644 --- a/experimental/schemabuilder/reflector.go +++ b/experimental/schemabuilder/reflector.go @@ -270,7 +270,7 @@ func (b *Builder) UpdateQueryDefinition(t *testing.T, outdir string) sdkapi.Quer body, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, schema, body) - panel := sdkapi.PseudoPanel{ + panel := pseudoPanel{ Type: "table", } panel.Targets = examplePanelTargets(&sdkapi.DataSourceRef{ diff --git a/v0alpha1/panel.go b/v0alpha1/panel.go deleted file mode 100644 index cb2f62e39..000000000 --- a/v0alpha1/panel.go +++ /dev/null @@ -1,18 +0,0 @@ -package v0alpha1 - -type PseudoPanel struct { - // Numeric panel id - ID int `json:"id,omitempty"` - - // The panel plugin type - Type string `json:"type"` - - // The panel title - Title string `json:"title,omitempty"` - - // This should no longer be necessary since each target has the datasource reference - Datasource *DataSourceRef `json:"datasource,omitempty"` - - // The query targets - Targets []DataQuery `json:"targets"` -} diff --git a/v0alpha1/query.go b/v0alpha1/query.go index e82014096..f20b11892 100644 --- a/v0alpha1/query.go +++ b/v0alpha1/query.go @@ -341,6 +341,9 @@ func (g *CommonQueryProperties) readQuery(iter *jsoniter.Iterator, return err } +// CommonQueryProperties are properties that can be added to all queries. +// These properties live in the same JSON level as datasource specific properties, +// so care must be taken to ensure they do not overlap type CommonQueryProperties struct { // RefID is the unique identifier of the query, set by the frontend call. RefID string `json:"refId,omitempty"` From e9b643e1d96952a410f817712409f1280617e8d7 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 4 Mar 2024 11:37:27 -0800 Subject: [PATCH 62/71] use k8s style paths --- apis/README.md | 7 +++++++ {v0alpha1 => apis/sdkapi/v0alpha1}/metaV1.go | 0 {v0alpha1 => apis/sdkapi/v0alpha1}/openapi.go | 8 ++++---- .../sdkapi/v0alpha1}/openapi_test.go | 0 .../v0alpha1}/query.definition.schema.json | 0 {v0alpha1 => apis/sdkapi/v0alpha1}/query.go | 13 ++----------- .../sdkapi/v0alpha1}/query.schema.json | 0 .../sdkapi/v0alpha1}/query_definition.go | 0 {v0alpha1 => apis/sdkapi/v0alpha1}/query_test.go | 16 ++++++++-------- {v0alpha1 => apis/sdkapi/v0alpha1}/schema.go | 0 .../sdkapi/v0alpha1}/schema_test.go | 0 .../v0alpha1}/testdata/sample_query_results.json | 0 .../sdkapi/v0alpha1}/unstructured.go | 0 experimental/schemabuilder/example/query_test.go | 2 +- experimental/schemabuilder/examples.go | 2 +- experimental/schemabuilder/panel.go | 6 +++--- experimental/schemabuilder/reflector.go | 12 ++---------- experimental/schemabuilder/reflector_test.go | 12 ++++++------ experimental/schemabuilder/schema.go | 2 +- 19 files changed, 35 insertions(+), 45 deletions(-) create mode 100644 apis/README.md rename {v0alpha1 => apis/sdkapi/v0alpha1}/metaV1.go (100%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/openapi.go (86%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/openapi_test.go (100%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/query.definition.schema.json (100%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/query.go (97%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/query.schema.json (100%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/query_definition.go (100%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/query_test.go (85%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/schema.go (100%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/schema_test.go (100%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/testdata/sample_query_results.json (100%) rename {v0alpha1 => apis/sdkapi/v0alpha1}/unstructured.go (100%) diff --git a/apis/README.md b/apis/README.md new file mode 100644 index 000000000..03d6bc350 --- /dev/null +++ b/apis/README.md @@ -0,0 +1,7 @@ +This package helps exposes objects from the plugin SDK into k8s based apis. + +This should most likely include the k8s.io/apimachinery dependency -- however this initial work +will just mock that values and see how well that works. Avoiding the need for https://github.com/grafana/grafana-plugin-sdk-go/pull/909 + + + diff --git a/v0alpha1/metaV1.go b/apis/sdkapi/v0alpha1/metaV1.go similarity index 100% rename from v0alpha1/metaV1.go rename to apis/sdkapi/v0alpha1/metaV1.go diff --git a/v0alpha1/openapi.go b/apis/sdkapi/v0alpha1/openapi.go similarity index 86% rename from v0alpha1/openapi.go rename to apis/sdkapi/v0alpha1/openapi.go index 2b22851bf..c504f38e6 100644 --- a/v0alpha1/openapi.go +++ b/apis/sdkapi/v0alpha1/openapi.go @@ -12,10 +12,10 @@ var f embed.FS func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ - "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), - "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), - "github.com/grafana/grafana-plugin-sdk-go/v0alpha1.DataQuery": schemaDataQuery(ref), - "github.com/grafana/grafana-plugin-sdk-go/v0alpha1.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), + "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), + "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), + "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1.DataQuery": schemaDataQuery(ref), + "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), } } diff --git a/v0alpha1/openapi_test.go b/apis/sdkapi/v0alpha1/openapi_test.go similarity index 100% rename from v0alpha1/openapi_test.go rename to apis/sdkapi/v0alpha1/openapi_test.go diff --git a/v0alpha1/query.definition.schema.json b/apis/sdkapi/v0alpha1/query.definition.schema.json similarity index 100% rename from v0alpha1/query.definition.schema.json rename to apis/sdkapi/v0alpha1/query.definition.schema.json diff --git a/v0alpha1/query.go b/apis/sdkapi/v0alpha1/query.go similarity index 97% rename from v0alpha1/query.go rename to apis/sdkapi/v0alpha1/query.go index f20b11892..e813cb251 100644 --- a/v0alpha1/query.go +++ b/apis/sdkapi/v0alpha1/query.go @@ -35,16 +35,6 @@ type DataQuery struct { additional map[string]any `json:"-"` // note this uses custom JSON marshalling } -// CommonProperties implements DataQuery. -func (g *DataQuery) CommonProperties() *CommonQueryProperties { - return &g.CommonQueryProperties -} - -// Dependencies implements DataQuery. -func (g *DataQuery) Dependencies() []string { - return nil -} - func NewDataQuery(body map[string]any) DataQuery { g := &DataQuery{ additional: make(map[string]any), @@ -162,9 +152,10 @@ func (g *DataQuery) Get(key string) (any, bool) { return v, ok } -func (g *DataQuery) MustString(key string) string { +func (g *DataQuery) GetString(key string) string { v, ok := g.Get(key) if ok { + // At the root convert to string s, ok := v.(string) if ok { return s diff --git a/v0alpha1/query.schema.json b/apis/sdkapi/v0alpha1/query.schema.json similarity index 100% rename from v0alpha1/query.schema.json rename to apis/sdkapi/v0alpha1/query.schema.json diff --git a/v0alpha1/query_definition.go b/apis/sdkapi/v0alpha1/query_definition.go similarity index 100% rename from v0alpha1/query_definition.go rename to apis/sdkapi/v0alpha1/query_definition.go diff --git a/v0alpha1/query_test.go b/apis/sdkapi/v0alpha1/query_test.go similarity index 85% rename from v0alpha1/query_test.go rename to apis/sdkapi/v0alpha1/query_test.go index c084ead40..0d2cefd03 100644 --- a/v0alpha1/query_test.go +++ b/apis/sdkapi/v0alpha1/query_test.go @@ -43,7 +43,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { t.Run("verify raw unmarshal", func(t *testing.T) { require.Len(t, req.Queries, 2) require.Equal(t, "b1808c48-9fc9-4045-82d7-081781f8a553", req.Queries[0].Datasource.UID) - require.Equal(t, "spreadsheetID", req.Queries[0].MustString("spreadsheet")) + require.Equal(t, "spreadsheetID", req.Queries[0].GetString("spreadsheet")) // Write the query (with additional spreadsheetID) to JSON out, err := json.MarshalIndent(req.Queries[0], "", " ") @@ -53,7 +53,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { query := &DataQuery{} err = json.Unmarshal(out, query) require.NoError(t, err) - require.Equal(t, "spreadsheetID", query.MustString("spreadsheet")) + require.Equal(t, "spreadsheetID", query.GetString("spreadsheet")) // The second query has an explicit time range, and legacy datasource name out, err = json.MarshalIndent(req.Queries[1], "", " ") @@ -92,21 +92,21 @@ func TestQueryBuilders(t *testing.T) { prop := "testkey" testQ1 := &DataQuery{} testQ1.Set(prop, "A") - require.Equal(t, "A", testQ1.MustString(prop)) + require.Equal(t, "A", testQ1.GetString(prop)) testQ1.Set(prop, "B") - require.Equal(t, "B", testQ1.MustString(prop)) + require.Equal(t, "B", testQ1.GetString(prop)) testQ2 := testQ1 testQ2.Set(prop, "C") - require.Equal(t, "C", testQ1.MustString(prop)) - require.Equal(t, "C", testQ2.MustString(prop)) + require.Equal(t, "C", testQ1.GetString(prop)) + require.Equal(t, "C", testQ2.GetString(prop)) // Uses the official field when exists testQ2.Set("queryType", "D") require.Equal(t, "D", testQ2.QueryType) require.Equal(t, "D", testQ1.QueryType) - require.Equal(t, "D", testQ2.MustString("queryType")) + require.Equal(t, "D", testQ2.GetString("queryType")) // Map constructor testQ3 := NewDataQuery(map[string]any{ @@ -114,5 +114,5 @@ func TestQueryBuilders(t *testing.T) { "extra": "E", }) require.Equal(t, "D", testQ3.QueryType) - require.Equal(t, "E", testQ3.MustString("extra")) + require.Equal(t, "E", testQ3.GetString("extra")) } diff --git a/v0alpha1/schema.go b/apis/sdkapi/v0alpha1/schema.go similarity index 100% rename from v0alpha1/schema.go rename to apis/sdkapi/v0alpha1/schema.go diff --git a/v0alpha1/schema_test.go b/apis/sdkapi/v0alpha1/schema_test.go similarity index 100% rename from v0alpha1/schema_test.go rename to apis/sdkapi/v0alpha1/schema_test.go diff --git a/v0alpha1/testdata/sample_query_results.json b/apis/sdkapi/v0alpha1/testdata/sample_query_results.json similarity index 100% rename from v0alpha1/testdata/sample_query_results.json rename to apis/sdkapi/v0alpha1/testdata/sample_query_results.json diff --git a/v0alpha1/unstructured.go b/apis/sdkapi/v0alpha1/unstructured.go similarity index 100% rename from v0alpha1/unstructured.go rename to apis/sdkapi/v0alpha1/unstructured.go diff --git a/experimental/schemabuilder/example/query_test.go b/experimental/schemabuilder/example/query_test.go index 7f5fea12c..797e00575 100644 --- a/experimental/schemabuilder/example/query_test.go +++ b/experimental/schemabuilder/example/query_test.go @@ -4,8 +4,8 @@ import ( "reflect" "testing" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" "github.com/stretchr/testify/require" ) diff --git a/experimental/schemabuilder/examples.go b/experimental/schemabuilder/examples.go index 7880bcc7f..37e78baa7 100644 --- a/experimental/schemabuilder/examples.go +++ b/experimental/schemabuilder/examples.go @@ -1,7 +1,7 @@ package schemabuilder import ( - sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" ) func exampleRequest(defs sdkapi.QueryTypeDefinitionList) sdkapi.DataQueryRequest { diff --git a/experimental/schemabuilder/panel.go b/experimental/schemabuilder/panel.go index 7d47f81bf..0b2edc6c2 100644 --- a/experimental/schemabuilder/panel.go +++ b/experimental/schemabuilder/panel.go @@ -1,6 +1,6 @@ package schemabuilder -import "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" +import sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" // This is only used to write out a sample panel query // It is not public and not intended to represent a real panel @@ -15,8 +15,8 @@ type pseudoPanel struct { Title string `json:"title,omitempty"` // This should no longer be necessary since each target has the datasource reference - Datasource *v0alpha1.DataSourceRef `json:"datasource,omitempty"` + Datasource *sdkapi.DataSourceRef `json:"datasource,omitempty"` // The query targets - Targets []v0alpha1.DataQuery `json:"targets"` + Targets []sdkapi.DataQuery `json:"targets"` } diff --git a/experimental/schemabuilder/reflector.go b/experimental/schemabuilder/reflector.go index c5a9a0f20..0745f1179 100644 --- a/experimental/schemabuilder/reflector.go +++ b/experimental/schemabuilder/reflector.go @@ -6,14 +6,13 @@ import ( "os" "path/filepath" "reflect" - "strings" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/data" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" "github.com/invopop/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -82,14 +81,7 @@ func NewSchemaBuilder(opts BuilderOptions) (*Builder, error) { r := new(jsonschema.Reflector) r.DoNotReference = true for _, scan := range opts.ScanCode { - base := scan.BasePackage - for _, v := range strings.Split(scan.CodePath, "/") { - if v == "" || v == "." || v == ".." { - continue - } - base += "/dummy" // fixes the resolution - } - if err := r.AddGoComments(base, scan.CodePath); err != nil { + if err := r.AddGoComments(scan.BasePackage, scan.CodePath); err != nil { return nil, err } } diff --git a/experimental/schemabuilder/reflector_test.go b/experimental/schemabuilder/reflector_test.go index a936300f4..75e64bce6 100644 --- a/experimental/schemabuilder/reflector_test.go +++ b/experimental/schemabuilder/reflector_test.go @@ -5,8 +5,8 @@ import ( "reflect" "testing" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/data" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" "github.com/invopop/jsonschema" "github.com/stretchr/testify/require" ) @@ -16,8 +16,8 @@ func TestWriteQuerySchema(t *testing.T) { PluginID: []string{"dummy"}, ScanCode: []CodePaths{ { - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/v0alpha1", - CodePath: "../../v0alpha1", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi", + CodePath: "../../apis/sdkapi/v0alpha1", }, { BasePackage: "github.com/grafana/grafana-plugin-sdk-go/data", @@ -40,7 +40,7 @@ func TestWriteQuerySchema(t *testing.T) { // // Hide this old property query.Properties.Delete("datasourceId") - outfile := "../../v0alpha1/query.schema.json" + outfile := "../../apis/sdkapi/v0alpha1/query.schema.json" old, _ := os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) @@ -54,10 +54,10 @@ func TestWriteQuerySchema(t *testing.T) { updateEnumDescriptions(query) query.ID = "" query.Version = draft04 // used by kube-openapi - outfile = "../../v0alpha1/query.definition.schema.json" + outfile = "../../apis/sdkapi/v0alpha1/query.definition.schema.json" old, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) - def := sdkapi.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/v0alpha1.QueryTypeDefinitionSpec"] + def := sdkapi.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1.QueryTypeDefinitionSpec"] require.Equal(t, query.Properties.Len(), len(def.Schema.Properties)) } diff --git a/experimental/schemabuilder/schema.go b/experimental/schemabuilder/schema.go index 1113cd56a..747a2f440 100644 --- a/experimental/schemabuilder/schema.go +++ b/experimental/schemabuilder/schema.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/v0alpha1" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" "k8s.io/kube-openapi/pkg/validation/spec" ) From c52d81c107da995ba0f4e320c1ab4f44d1bc8f11 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 4 Mar 2024 12:02:12 -0800 Subject: [PATCH 63/71] use codegen --- apis/sdkapi/v0alpha1/query.go | 49 ++++++++---------------- apis/sdkapi/v0alpha1/query_definition.go | 20 +--------- 2 files changed, 16 insertions(+), 53 deletions(-) diff --git a/apis/sdkapi/v0alpha1/query.go b/apis/sdkapi/v0alpha1/query.go index e813cb251..4b54aaf69 100644 --- a/apis/sdkapi/v0alpha1/query.go +++ b/apis/sdkapi/v0alpha1/query.go @@ -45,40 +45,6 @@ func NewDataQuery(body map[string]any) DataQuery { return *g } -func (g *DataQuery) DeepCopy() *DataQuery { - if g == nil { - return nil - } - out := new(DataQuery) - jj, err := json.Marshal(g) - if err == nil { - _ = json.Unmarshal(jj, out) - } - return out -} - -func (g *DataQuery) DeepCopyInto(out *DataQuery) { - clone := g.DeepCopy() - *out = *clone -} - -func (g *DataQueryRequest) DeepCopy() *DataQueryRequest { - if g == nil { - return nil - } - out := new(DataQueryRequest) - jj, err := json.Marshal(g) - if err == nil { - _ = json.Unmarshal(jj, out) - } - return out -} - -func (g *DataQueryRequest) DeepCopyInto(out *DataQueryRequest) { - clone := g.DeepCopy() - *out = *clone -} - // Set allows setting values using key/value pairs func (g *DataQuery) Set(key string, val any) *DataQuery { switch key { @@ -207,6 +173,21 @@ func (g *DataQuery) UnmarshalJSON(b []byte) error { return g.readQuery(iter) } +func (in *DataQuery) DeepCopyInto(out *DataQuery) { + *out = *in + in.CommonQueryProperties.DeepCopyInto(&out.CommonQueryProperties) + if in.additional != nil { + out.additional = map[string]any{} + if len(in.additional) > 0 { + jj, err := json.Marshal(in.additional) + if err != nil { + _ = json.Unmarshal(jj, &out.additional) + } + } + } + return +} + func writeQuery(g *DataQuery, stream *j.Stream) { q := g.CommonQueryProperties stream.WriteObjectStart() diff --git a/apis/sdkapi/v0alpha1/query_definition.go b/apis/sdkapi/v0alpha1/query_definition.go index e5e977bac..91662f7b5 100644 --- a/apis/sdkapi/v0alpha1/query_definition.go +++ b/apis/sdkapi/v0alpha1/query_definition.go @@ -1,13 +1,12 @@ package v0alpha1 import ( - "encoding/json" "fmt" ) // QueryTypeDefinition is a kubernetes shaped object that represents a single query definition type QueryTypeDefinition struct { - ObjectMeta ObjectMeta `json:"metadata,omitempty"` + ObjectMeta `json:"metadata,omitempty"` Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` } @@ -43,23 +42,6 @@ type QueryTypeDefinitionSpec struct { Changelog []string `json:"changelog,omitempty"` } -func (g *QueryTypeDefinitionSpec) DeepCopy() *QueryTypeDefinitionSpec { - if g == nil { - return nil - } - out := new(QueryTypeDefinitionSpec) - jj, err := json.Marshal(g) - if err == nil { - _ = json.Unmarshal(jj, out) - } - return out -} - -func (g *QueryTypeDefinitionSpec) DeepCopyInto(out *QueryTypeDefinitionSpec) { - clone := g.DeepCopy() - *out = *clone -} - type QueryExample struct { // Version identifier or empty if only one exists Name string `json:"name,omitempty"` From 91df7c56a5148a2dc9e414882c9890251d273296 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 4 Mar 2024 12:02:18 -0800 Subject: [PATCH 64/71] use codegen --- apis/sdkapi/v0alpha1/zz_generated.deepcopy.go | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 apis/sdkapi/v0alpha1/zz_generated.deepcopy.go diff --git a/apis/sdkapi/v0alpha1/zz_generated.deepcopy.go b/apis/sdkapi/v0alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..14493f860 --- /dev/null +++ b/apis/sdkapi/v0alpha1/zz_generated.deepcopy.go @@ -0,0 +1,187 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v0alpha1 + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommonQueryProperties) DeepCopyInto(out *CommonQueryProperties) { + *out = *in + if in.ResultAssertions != nil { + in, out := &in.ResultAssertions, &out.ResultAssertions + *out = new(ResultAssertions) + (*in).DeepCopyInto(*out) + } + if in.TimeRange != nil { + in, out := &in.TimeRange, &out.TimeRange + *out = new(TimeRange) + **out = **in + } + if in.Datasource != nil { + in, out := &in.Datasource, &out.Datasource + *out = new(DataSourceRef) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommonQueryProperties. +func (in *CommonQueryProperties) DeepCopy() *CommonQueryProperties { + if in == nil { + return nil + } + out := new(CommonQueryProperties) + in.DeepCopyInto(out) + return out +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataQuery. +func (in *DataQuery) DeepCopy() *DataQuery { + if in == nil { + return nil + } + out := new(DataQuery) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataQueryRequest) DeepCopyInto(out *DataQueryRequest) { + *out = *in + out.TimeRange = in.TimeRange + if in.Queries != nil { + in, out := &in.Queries, &out.Queries + *out = make([]DataQuery, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataQueryRequest. +func (in *DataQueryRequest) DeepCopy() *DataQueryRequest { + if in == nil { + return nil + } + out := new(DataQueryRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataSourceRef) DeepCopyInto(out *DataSourceRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataSourceRef. +func (in *DataSourceRef) DeepCopy() *DataSourceRef { + if in == nil { + return nil + } + out := new(DataSourceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DiscriminatorFieldValue) DeepCopyInto(out *DiscriminatorFieldValue) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiscriminatorFieldValue. +func (in *DiscriminatorFieldValue) DeepCopy() *DiscriminatorFieldValue { + if in == nil { + return nil + } + out := new(DiscriminatorFieldValue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueryExample) DeepCopyInto(out *QueryExample) { + *out = *in + in.SaveModel.DeepCopyInto(&out.SaveModel) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryExample. +func (in *QueryExample) DeepCopy() *QueryExample { + if in == nil { + return nil + } + out := new(QueryExample) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueryTypeDefinitionSpec) DeepCopyInto(out *QueryTypeDefinitionSpec) { + *out = *in + if in.Discriminators != nil { + in, out := &in.Discriminators, &out.Discriminators + *out = make([]DiscriminatorFieldValue, len(*in)) + copy(*out, *in) + } + in.Schema.DeepCopyInto(&out.Schema) + if in.Examples != nil { + in, out := &in.Examples, &out.Examples + *out = make([]QueryExample, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Changelog != nil { + in, out := &in.Changelog, &out.Changelog + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryTypeDefinitionSpec. +func (in *QueryTypeDefinitionSpec) DeepCopy() *QueryTypeDefinitionSpec { + if in == nil { + return nil + } + out := new(QueryTypeDefinitionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResultAssertions) DeepCopyInto(out *ResultAssertions) { + *out = *in + out.TypeVersion = in.TypeVersion + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResultAssertions. +func (in *ResultAssertions) DeepCopy() *ResultAssertions { + if in == nil { + return nil + } + out := new(ResultAssertions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TimeRange) DeepCopyInto(out *TimeRange) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeRange. +func (in *TimeRange) DeepCopy() *TimeRange { + if in == nil { + return nil + } + out := new(TimeRange) + in.DeepCopyInto(out) + return out +} From 66438c49736606c8c3bac566a60a3d34b15f6ba4 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 4 Mar 2024 12:24:54 -0800 Subject: [PATCH 65/71] lint --- apis/sdkapi/v0alpha1/query.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/apis/sdkapi/v0alpha1/query.go b/apis/sdkapi/v0alpha1/query.go index 4b54aaf69..d8550e49b 100644 --- a/apis/sdkapi/v0alpha1/query.go +++ b/apis/sdkapi/v0alpha1/query.go @@ -173,19 +173,18 @@ func (g *DataQuery) UnmarshalJSON(b []byte) error { return g.readQuery(iter) } -func (in *DataQuery) DeepCopyInto(out *DataQuery) { - *out = *in - in.CommonQueryProperties.DeepCopyInto(&out.CommonQueryProperties) - if in.additional != nil { +func (g *DataQuery) DeepCopyInto(out *DataQuery) { + *out = *g + g.CommonQueryProperties.DeepCopyInto(&out.CommonQueryProperties) + if g.additional != nil { out.additional = map[string]any{} - if len(in.additional) > 0 { - jj, err := json.Marshal(in.additional) + if len(g.additional) > 0 { + jj, err := json.Marshal(g.additional) if err != nil { _ = json.Unmarshal(jj, &out.additional) } } } - return } func writeQuery(g *DataQuery, stream *j.Stream) { From 557e2083f12c59a19712c6ef8940c88e78d2a4e0 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 5 Mar 2024 13:38:58 -0800 Subject: [PATCH 66/71] rename to data --- apis/data/v0alpha1/client.go | 66 +++++++++++++++++++ apis/data/v0alpha1/client_test.go | 52 +++++++++++++++ apis/{sdkapi => data}/v0alpha1/metaV1.go | 0 apis/{sdkapi => data}/v0alpha1/openapi.go | 8 +-- .../{sdkapi => data}/v0alpha1/openapi_test.go | 0 .../v0alpha1/query.definition.schema.json | 0 apis/{sdkapi => data}/v0alpha1/query.go | 2 +- .../v0alpha1/query.schema.json | 0 .../v0alpha1/query_definition.go | 0 apis/{sdkapi => data}/v0alpha1/query_test.go | 4 +- apis/{sdkapi => data}/v0alpha1/schema.go | 0 apis/{sdkapi => data}/v0alpha1/schema_test.go | 0 .../testdata/sample_query_results.json | 0 .../{sdkapi => data}/v0alpha1/unstructured.go | 0 .../v0alpha1/zz_generated.deepcopy.go | 8 +-- .../schemabuilder/example/query_test.go | 20 +++--- experimental/schemabuilder/examples.go | 18 ++--- experimental/schemabuilder/panel.go | 2 +- experimental/schemabuilder/reflector.go | 2 +- experimental/schemabuilder/reflector_test.go | 12 ++-- experimental/schemabuilder/schema.go | 2 +- 21 files changed, 157 insertions(+), 39 deletions(-) create mode 100644 apis/data/v0alpha1/client.go create mode 100644 apis/data/v0alpha1/client_test.go rename apis/{sdkapi => data}/v0alpha1/metaV1.go (100%) rename apis/{sdkapi => data}/v0alpha1/openapi.go (86%) rename apis/{sdkapi => data}/v0alpha1/openapi_test.go (100%) rename apis/{sdkapi => data}/v0alpha1/query.definition.schema.json (100%) rename apis/{sdkapi => data}/v0alpha1/query.go (99%) rename apis/{sdkapi => data}/v0alpha1/query.schema.json (100%) rename apis/{sdkapi => data}/v0alpha1/query_definition.go (100%) rename apis/{sdkapi => data}/v0alpha1/query_test.go (98%) rename apis/{sdkapi => data}/v0alpha1/schema.go (100%) rename apis/{sdkapi => data}/v0alpha1/schema_test.go (100%) rename apis/{sdkapi => data}/v0alpha1/testdata/sample_query_results.json (100%) rename apis/{sdkapi => data}/v0alpha1/unstructured.go (100%) rename apis/{sdkapi => data}/v0alpha1/zz_generated.deepcopy.go (96%) diff --git a/apis/data/v0alpha1/client.go b/apis/data/v0alpha1/client.go new file mode 100644 index 000000000..5a5751cce --- /dev/null +++ b/apis/data/v0alpha1/client.go @@ -0,0 +1,66 @@ +package v0alpha1 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" +) + +type QueryDataClient interface { + QueryData(ctx context.Context, req QueryDataRequest, headers ...string) (int, *backend.QueryDataResponse, error) +} + +type simpleHTTPClient struct { + url string + client *http.Client + headers []string +} + +func NewQueryDataClient(url string, client *http.Client, headers ...string) QueryDataClient { + if client == nil { + client = http.DefaultClient + } + return &simpleHTTPClient{ + url: url, + client: client, + headers: headers, + } +} + +func (c *simpleHTTPClient) QueryData(ctx context.Context, query QueryDataRequest, headers ...string) (int, *backend.QueryDataResponse, error) { + body, err := json.Marshal(query) + if err != nil { + return http.StatusBadRequest, nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewBuffer(body)) + if err != nil { + return http.StatusBadRequest, nil, err + } + headers = append(c.headers, headers...) + if (len(headers) % 2) != 0 { + return http.StatusBadRequest, nil, fmt.Errorf("headers must be in pairs of two") + } + for i := 0; i < len(headers); i += 2 { + req.Header.Set(headers[i], headers[i+1]) + } + req.Header.Set("Content-Type", "application/json") + + rsp, err := c.client.Do(req) + if err != nil { + return rsp.StatusCode, nil, err + } + defer rsp.Body.Close() + + qdr := &backend.QueryDataResponse{} + iter, err := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, rsp.Body, 1024*10) + if err == nil { + err = iter.ReadVal(qdr) + } + return rsp.StatusCode, qdr, err +} diff --git a/apis/data/v0alpha1/client_test.go b/apis/data/v0alpha1/client_test.go new file mode 100644 index 000000000..5079293c5 --- /dev/null +++ b/apis/data/v0alpha1/client_test.go @@ -0,0 +1,52 @@ +package v0alpha1_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" + "github.com/stretchr/testify/require" +) + +func TestQueryClient(t *testing.T) { + t.Skip() + + client := v0alpha1.NewQueryDataClient("http://localhost:3000/api/ds/query", nil, + "Authorization", "Bearer YOURKEYHERE", + ) + body := `{ + "from": "", + "to": "", + "queries": [ + { + "refId": "X", + "scenarioId": "csv_content", + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "PD8C576611E62080A" + }, + "csvContent": "a,b,c\n1,hello,true", + "hide": true + } + ] + }` + qdr := v0alpha1.QueryDataRequest{} + err := json.Unmarshal([]byte(body), &qdr) + require.NoError(t, err) + + code, rsp, err := client.QueryData(context.Background(), qdr) + require.NoError(t, err) + require.Equal(t, http.StatusOK, code) + + r, ok := rsp.Responses["X"] + require.True(t, ok) + + for _, frame := range r.Frames { + txt, err := frame.StringTable(20, 10) + require.NoError(t, err) + fmt.Printf("%s\n", txt) + } +} diff --git a/apis/sdkapi/v0alpha1/metaV1.go b/apis/data/v0alpha1/metaV1.go similarity index 100% rename from apis/sdkapi/v0alpha1/metaV1.go rename to apis/data/v0alpha1/metaV1.go diff --git a/apis/sdkapi/v0alpha1/openapi.go b/apis/data/v0alpha1/openapi.go similarity index 86% rename from apis/sdkapi/v0alpha1/openapi.go rename to apis/data/v0alpha1/openapi.go index c504f38e6..76366aabf 100644 --- a/apis/sdkapi/v0alpha1/openapi.go +++ b/apis/data/v0alpha1/openapi.go @@ -12,10 +12,10 @@ var f embed.FS func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ - "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), - "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), - "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1.DataQuery": schemaDataQuery(ref), - "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), + "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), + "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), + "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1.DataQuery": schemaDataQuery(ref), + "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), } } diff --git a/apis/sdkapi/v0alpha1/openapi_test.go b/apis/data/v0alpha1/openapi_test.go similarity index 100% rename from apis/sdkapi/v0alpha1/openapi_test.go rename to apis/data/v0alpha1/openapi_test.go diff --git a/apis/sdkapi/v0alpha1/query.definition.schema.json b/apis/data/v0alpha1/query.definition.schema.json similarity index 100% rename from apis/sdkapi/v0alpha1/query.definition.schema.json rename to apis/data/v0alpha1/query.definition.schema.json diff --git a/apis/sdkapi/v0alpha1/query.go b/apis/data/v0alpha1/query.go similarity index 99% rename from apis/sdkapi/v0alpha1/query.go rename to apis/data/v0alpha1/query.go index d8550e49b..a382f0ded 100644 --- a/apis/sdkapi/v0alpha1/query.go +++ b/apis/data/v0alpha1/query.go @@ -16,7 +16,7 @@ func init() { //nolint:gochecknoinits jsoniter.RegisterTypeDecoder("v0alpha1.DataQuery", &genericQueryCodec{}) } -type DataQueryRequest struct { +type QueryDataRequest struct { // Time range applied to each query (when not included in the query body) TimeRange `json:",inline"` diff --git a/apis/sdkapi/v0alpha1/query.schema.json b/apis/data/v0alpha1/query.schema.json similarity index 100% rename from apis/sdkapi/v0alpha1/query.schema.json rename to apis/data/v0alpha1/query.schema.json diff --git a/apis/sdkapi/v0alpha1/query_definition.go b/apis/data/v0alpha1/query_definition.go similarity index 100% rename from apis/sdkapi/v0alpha1/query_definition.go rename to apis/data/v0alpha1/query_definition.go diff --git a/apis/sdkapi/v0alpha1/query_test.go b/apis/data/v0alpha1/query_test.go similarity index 98% rename from apis/sdkapi/v0alpha1/query_test.go rename to apis/data/v0alpha1/query_test.go index 0d2cefd03..51ba48ab9 100644 --- a/apis/sdkapi/v0alpha1/query_test.go +++ b/apis/data/v0alpha1/query_test.go @@ -36,7 +36,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { "to": "1692646267389" }`) - req := &DataQueryRequest{} + req := &QueryDataRequest{} err := json.Unmarshal(request, req) require.NoError(t, err) @@ -74,7 +74,7 @@ func TestParseQueriesIntoQueryDataRequest(t *testing.T) { }) t.Run("same results from either parser", func(t *testing.T) { - typed := &DataQueryRequest{} + typed := &QueryDataRequest{} err = json.Unmarshal(request, typed) require.NoError(t, err) diff --git a/apis/sdkapi/v0alpha1/schema.go b/apis/data/v0alpha1/schema.go similarity index 100% rename from apis/sdkapi/v0alpha1/schema.go rename to apis/data/v0alpha1/schema.go diff --git a/apis/sdkapi/v0alpha1/schema_test.go b/apis/data/v0alpha1/schema_test.go similarity index 100% rename from apis/sdkapi/v0alpha1/schema_test.go rename to apis/data/v0alpha1/schema_test.go diff --git a/apis/sdkapi/v0alpha1/testdata/sample_query_results.json b/apis/data/v0alpha1/testdata/sample_query_results.json similarity index 100% rename from apis/sdkapi/v0alpha1/testdata/sample_query_results.json rename to apis/data/v0alpha1/testdata/sample_query_results.json diff --git a/apis/sdkapi/v0alpha1/unstructured.go b/apis/data/v0alpha1/unstructured.go similarity index 100% rename from apis/sdkapi/v0alpha1/unstructured.go rename to apis/data/v0alpha1/unstructured.go diff --git a/apis/sdkapi/v0alpha1/zz_generated.deepcopy.go b/apis/data/v0alpha1/zz_generated.deepcopy.go similarity index 96% rename from apis/sdkapi/v0alpha1/zz_generated.deepcopy.go rename to apis/data/v0alpha1/zz_generated.deepcopy.go index 14493f860..aaf8295c5 100644 --- a/apis/sdkapi/v0alpha1/zz_generated.deepcopy.go +++ b/apis/data/v0alpha1/zz_generated.deepcopy.go @@ -47,7 +47,7 @@ func (in *DataQuery) DeepCopy() *DataQuery { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DataQueryRequest) DeepCopyInto(out *DataQueryRequest) { +func (in *QueryDataRequest) DeepCopyInto(out *QueryDataRequest) { *out = *in out.TimeRange = in.TimeRange if in.Queries != nil { @@ -60,12 +60,12 @@ func (in *DataQueryRequest) DeepCopyInto(out *DataQueryRequest) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataQueryRequest. -func (in *DataQueryRequest) DeepCopy() *DataQueryRequest { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryDataRequest. +func (in *QueryDataRequest) DeepCopy() *QueryDataRequest { if in == nil { return nil } - out := new(DataQueryRequest) + out := new(QueryDataRequest) in.DeepCopyInto(out) return out } diff --git a/experimental/schemabuilder/example/query_test.go b/experimental/schemabuilder/example/query_test.go index 797e00575..edd7157c5 100644 --- a/experimental/schemabuilder/example/query_test.go +++ b/experimental/schemabuilder/example/query_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" + data "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder" "github.com/stretchr/testify/require" ) @@ -23,30 +23,30 @@ func TestQueryTypeDefinitions(t *testing.T) { }) require.NoError(t, err) err = builder.AddQueries(schemabuilder.QueryTypeInfo{ - Discriminators: sdkapi.NewDiscriminators("queryType", QueryTypeMath), + Discriminators: data.NewDiscriminators("queryType", QueryTypeMath), GoType: reflect.TypeOf(&MathQuery{}), - Examples: []sdkapi.QueryExample{ + Examples: []data.QueryExample{ { Name: "constant addition", - SaveModel: sdkapi.AsUnstructured(MathQuery{ + SaveModel: data.AsUnstructured(MathQuery{ Expression: "$A + 11", }), }, { Name: "math with two queries", - SaveModel: sdkapi.AsUnstructured(MathQuery{ + SaveModel: data.AsUnstructured(MathQuery{ Expression: "$A - $B", }), }, }, }, schemabuilder.QueryTypeInfo{ - Discriminators: sdkapi.NewDiscriminators("queryType", QueryTypeReduce), + Discriminators: data.NewDiscriminators("queryType", QueryTypeReduce), GoType: reflect.TypeOf(&ReduceQuery{}), - Examples: []sdkapi.QueryExample{ + Examples: []data.QueryExample{ { Name: "get max value", - SaveModel: sdkapi.AsUnstructured(ReduceQuery{ + SaveModel: data.AsUnstructured(ReduceQuery{ Expression: "$A", Reducer: ReducerMax, Settings: ReduceSettings{ @@ -57,9 +57,9 @@ func TestQueryTypeDefinitions(t *testing.T) { }, }, schemabuilder.QueryTypeInfo{ - Discriminators: sdkapi.NewDiscriminators("queryType", QueryTypeResample), + Discriminators: data.NewDiscriminators("queryType", QueryTypeResample), GoType: reflect.TypeOf(&ResampleQuery{}), - Examples: []sdkapi.QueryExample{}, + Examples: []data.QueryExample{}, }) require.NoError(t, err) diff --git a/experimental/schemabuilder/examples.go b/experimental/schemabuilder/examples.go index 37e78baa7..ebaffcb8d 100644 --- a/experimental/schemabuilder/examples.go +++ b/experimental/schemabuilder/examples.go @@ -1,22 +1,22 @@ package schemabuilder import ( - sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" + data "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" ) -func exampleRequest(defs sdkapi.QueryTypeDefinitionList) sdkapi.DataQueryRequest { - rsp := sdkapi.DataQueryRequest{ - TimeRange: sdkapi.TimeRange{ +func exampleRequest(defs data.QueryTypeDefinitionList) data.QueryDataRequest { + rsp := data.QueryDataRequest{ + TimeRange: data.TimeRange{ From: "now-1h", To: "now", }, - Queries: []sdkapi.DataQuery{}, + Queries: []data.DataQuery{}, } for _, def := range defs.Items { for _, sample := range def.Spec.Examples { if sample.SaveModel.Object != nil { - q := sdkapi.NewDataQuery(sample.SaveModel.Object) + q := data.NewDataQuery(sample.SaveModel.Object) q.RefID = string(rune('A' + len(rsp.Queries))) for _, dis := range def.Spec.Discriminators { _ = q.Set(dis.Field, dis.Value) @@ -36,13 +36,13 @@ func exampleRequest(defs sdkapi.QueryTypeDefinitionList) sdkapi.DataQueryRequest return rsp } -func examplePanelTargets(ds *sdkapi.DataSourceRef, defs sdkapi.QueryTypeDefinitionList) []sdkapi.DataQuery { - targets := []sdkapi.DataQuery{} +func examplePanelTargets(ds *data.DataSourceRef, defs data.QueryTypeDefinitionList) []data.DataQuery { + targets := []data.DataQuery{} for _, def := range defs.Items { for _, sample := range def.Spec.Examples { if sample.SaveModel.Object != nil { - q := sdkapi.NewDataQuery(sample.SaveModel.Object) + q := data.NewDataQuery(sample.SaveModel.Object) q.Datasource = ds q.RefID = string(rune('A' + len(targets))) for _, dis := range def.Spec.Discriminators { diff --git a/experimental/schemabuilder/panel.go b/experimental/schemabuilder/panel.go index 0b2edc6c2..4cb591c52 100644 --- a/experimental/schemabuilder/panel.go +++ b/experimental/schemabuilder/panel.go @@ -1,6 +1,6 @@ package schemabuilder -import sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" +import sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" // This is only used to write out a sample panel query // It is not public and not intended to represent a real panel diff --git a/experimental/schemabuilder/reflector.go b/experimental/schemabuilder/reflector.go index 0745f1179..b93678fec 100644 --- a/experimental/schemabuilder/reflector.go +++ b/experimental/schemabuilder/reflector.go @@ -11,7 +11,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/invopop/jsonschema" "github.com/stretchr/testify/assert" diff --git a/experimental/schemabuilder/reflector_test.go b/experimental/schemabuilder/reflector_test.go index 75e64bce6..748b163f5 100644 --- a/experimental/schemabuilder/reflector_test.go +++ b/experimental/schemabuilder/reflector_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/invopop/jsonschema" "github.com/stretchr/testify/require" @@ -16,8 +16,8 @@ func TestWriteQuerySchema(t *testing.T) { PluginID: []string{"dummy"}, ScanCode: []CodePaths{ { - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi", - CodePath: "../../apis/sdkapi/v0alpha1", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/apis/data", + CodePath: "../../apis/data/v0alpha1", }, { BasePackage: "github.com/grafana/grafana-plugin-sdk-go/data", @@ -40,7 +40,7 @@ func TestWriteQuerySchema(t *testing.T) { // // Hide this old property query.Properties.Delete("datasourceId") - outfile := "../../apis/sdkapi/v0alpha1/query.schema.json" + outfile := "../../apis/data/v0alpha1/query.schema.json" old, _ := os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) @@ -54,10 +54,10 @@ func TestWriteQuerySchema(t *testing.T) { updateEnumDescriptions(query) query.ID = "" query.Version = draft04 // used by kube-openapi - outfile = "../../apis/sdkapi/v0alpha1/query.definition.schema.json" + outfile = "../../apis/data/v0alpha1/query.definition.schema.json" old, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) - def := sdkapi.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1.QueryTypeDefinitionSpec"] + def := sdkapi.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1.QueryTypeDefinitionSpec"] require.Equal(t, query.Properties.Len(), len(def.Schema.Properties)) } diff --git a/experimental/schemabuilder/schema.go b/experimental/schemabuilder/schema.go index 747a2f440..48310d9a7 100644 --- a/experimental/schemabuilder/schema.go +++ b/experimental/schemabuilder/schema.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/sdkapi/v0alpha1" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" "k8s.io/kube-openapi/pkg/validation/spec" ) From 283c5df71d88a33c2dc10bc23fecbaa1989c9f3b Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 5 Mar 2024 23:18:13 -0800 Subject: [PATCH 67/71] update readme --- apis/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/apis/README.md b/apis/README.md index 03d6bc350..9c78a30ba 100644 --- a/apis/README.md +++ b/apis/README.md @@ -1,7 +1,15 @@ -This package helps exposes objects from the plugin SDK into k8s based apis. +== APIServer APIs -This should most likely include the k8s.io/apimachinery dependency -- however this initial work -will just mock that values and see how well that works. Avoiding the need for https://github.com/grafana/grafana-plugin-sdk-go/pull/909 +This package aims to expose types from the plugins-sdk in the grafana apiserver. +Currently, the types are not useable directly so we can avoid adding a dependency on k8s.io/apimachinery +until it is more necessary. See https://github.com/grafana/grafana-plugin-sdk-go/pull/909 +The "v0alpha1" version should be considered experimental and is subject to change at any time without notice. +Once it is more stable, it will be released as a versioned API (v1) + +=== Codegen + +The file [apis/data/v0alpha1/zz_generated.deepcopy.go](data/v0alpha1/zz_generated.deepcopy.go) was generated by copying the folder structure into +https://github.com/grafana/grafana/tree/main/pkg/apis and then running the [hack scripts](https://github.com/grafana/grafana/tree/v10.3.3/hack) in /hack/update-codegen.sh \ No newline at end of file From 782e1f50eaa538d9c08823b6497a8f29038dde99 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 5 Mar 2024 23:26:19 -0800 Subject: [PATCH 68/71] update headers --- apis/README.md | 6 +++--- apis/data/v0alpha1/client.go | 17 ++++++----------- apis/data/v0alpha1/client_test.go | 5 +++-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/apis/README.md b/apis/README.md index 9c78a30ba..33fd54a65 100644 --- a/apis/README.md +++ b/apis/README.md @@ -1,4 +1,4 @@ -== APIServer APIs +## APIServer APIs This package aims to expose types from the plugins-sdk in the grafana apiserver. @@ -9,7 +9,7 @@ The "v0alpha1" version should be considered experimental and is subject to chang Once it is more stable, it will be released as a versioned API (v1) -=== Codegen +### Codegen The file [apis/data/v0alpha1/zz_generated.deepcopy.go](data/v0alpha1/zz_generated.deepcopy.go) was generated by copying the folder structure into -https://github.com/grafana/grafana/tree/main/pkg/apis and then running the [hack scripts](https://github.com/grafana/grafana/tree/v10.3.3/hack) in /hack/update-codegen.sh \ No newline at end of file +https://github.com/grafana/grafana/tree/main/pkg/apis and then run `hack/update-codegen.sh data` in [hack scripts](https://github.com/grafana/grafana/tree/v10.3.3/hack). \ No newline at end of file diff --git a/apis/data/v0alpha1/client.go b/apis/data/v0alpha1/client.go index 5a5751cce..b84ae196a 100644 --- a/apis/data/v0alpha1/client.go +++ b/apis/data/v0alpha1/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "fmt" "net/http" "github.com/grafana/grafana-plugin-sdk-go/backend" @@ -12,16 +11,16 @@ import ( ) type QueryDataClient interface { - QueryData(ctx context.Context, req QueryDataRequest, headers ...string) (int, *backend.QueryDataResponse, error) + QueryData(ctx context.Context, req QueryDataRequest) (int, *backend.QueryDataResponse, error) } type simpleHTTPClient struct { url string client *http.Client - headers []string + headers map[string]string } -func NewQueryDataClient(url string, client *http.Client, headers ...string) QueryDataClient { +func NewQueryDataClient(url string, client *http.Client, headers map[string]string) QueryDataClient { if client == nil { client = http.DefaultClient } @@ -32,7 +31,7 @@ func NewQueryDataClient(url string, client *http.Client, headers ...string) Quer } } -func (c *simpleHTTPClient) QueryData(ctx context.Context, query QueryDataRequest, headers ...string) (int, *backend.QueryDataResponse, error) { +func (c *simpleHTTPClient) QueryData(ctx context.Context, query QueryDataRequest) (int, *backend.QueryDataResponse, error) { body, err := json.Marshal(query) if err != nil { return http.StatusBadRequest, nil, err @@ -42,12 +41,8 @@ func (c *simpleHTTPClient) QueryData(ctx context.Context, query QueryDataRequest if err != nil { return http.StatusBadRequest, nil, err } - headers = append(c.headers, headers...) - if (len(headers) % 2) != 0 { - return http.StatusBadRequest, nil, fmt.Errorf("headers must be in pairs of two") - } - for i := 0; i < len(headers); i += 2 { - req.Header.Set(headers[i], headers[i+1]) + for k, v := range c.headers { + req.Header.Set(k, v) } req.Header.Set("Content-Type", "application/json") diff --git a/apis/data/v0alpha1/client_test.go b/apis/data/v0alpha1/client_test.go index 5079293c5..dd5eb125f 100644 --- a/apis/data/v0alpha1/client_test.go +++ b/apis/data/v0alpha1/client_test.go @@ -15,8 +15,9 @@ func TestQueryClient(t *testing.T) { t.Skip() client := v0alpha1.NewQueryDataClient("http://localhost:3000/api/ds/query", nil, - "Authorization", "Bearer YOURKEYHERE", - ) + map[string]string{ + "Authorization": "Bearer XYZ", + }) body := `{ "from": "", "to": "", From 342bf4e354bc0513eaec5c4c5e07bb8b717d2ef6 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 5 Mar 2024 23:26:29 -0800 Subject: [PATCH 69/71] update headers --- apis/data/v0alpha1/doc.go | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 apis/data/v0alpha1/doc.go diff --git a/apis/data/v0alpha1/doc.go b/apis/data/v0alpha1/doc.go new file mode 100644 index 000000000..6a049ca39 --- /dev/null +++ b/apis/data/v0alpha1/doc.go @@ -0,0 +1,6 @@ +// +k8s:deepcopy-gen=package +// +k8s:openapi-gen=true +// +k8s:defaulter-gen=TypeMeta +// +groupName=data.grafana.com + +package v0alpha1 From 86b43fab17b0886c7e896b6ec30cdb3c14f65392 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 6 Mar 2024 06:45:11 -0800 Subject: [PATCH 70/71] move package --- apis/README.md | 15 - apis/data/v0alpha1/client.go | 61 --- apis/data/v0alpha1/client_test.go | 53 --- apis/data/v0alpha1/doc.go | 6 - apis/data/v0alpha1/metaV1.go | 19 - apis/data/v0alpha1/openapi.go | 82 ---- apis/data/v0alpha1/openapi_test.go | 40 -- .../v0alpha1/query.definition.schema.json | 72 --- apis/data/v0alpha1/query.go | 419 ------------------ apis/data/v0alpha1/query.schema.json | 114 ----- apis/data/v0alpha1/query_definition.go | 78 ---- apis/data/v0alpha1/query_test.go | 118 ----- apis/data/v0alpha1/schema.go | 70 --- apis/data/v0alpha1/schema_test.go | 31 -- .../testdata/sample_query_results.json | 51 --- apis/data/v0alpha1/unstructured.go | 81 ---- apis/data/v0alpha1/zz_generated.deepcopy.go | 187 -------- .../schemabuilder/example/query_test.go | 2 +- experimental/schemabuilder/examples.go | 2 +- experimental/schemabuilder/panel.go | 2 +- experimental/schemabuilder/reflector.go | 2 +- experimental/schemabuilder/reflector_test.go | 18 +- experimental/schemabuilder/schema.go | 2 +- experimental/testdata/folder.golden.txt | 36 +- 24 files changed, 32 insertions(+), 1529 deletions(-) delete mode 100644 apis/README.md delete mode 100644 apis/data/v0alpha1/client.go delete mode 100644 apis/data/v0alpha1/client_test.go delete mode 100644 apis/data/v0alpha1/doc.go delete mode 100644 apis/data/v0alpha1/metaV1.go delete mode 100644 apis/data/v0alpha1/openapi.go delete mode 100644 apis/data/v0alpha1/openapi_test.go delete mode 100644 apis/data/v0alpha1/query.definition.schema.json delete mode 100644 apis/data/v0alpha1/query.go delete mode 100644 apis/data/v0alpha1/query.schema.json delete mode 100644 apis/data/v0alpha1/query_definition.go delete mode 100644 apis/data/v0alpha1/query_test.go delete mode 100644 apis/data/v0alpha1/schema.go delete mode 100644 apis/data/v0alpha1/schema_test.go delete mode 100644 apis/data/v0alpha1/testdata/sample_query_results.json delete mode 100644 apis/data/v0alpha1/unstructured.go delete mode 100644 apis/data/v0alpha1/zz_generated.deepcopy.go diff --git a/apis/README.md b/apis/README.md deleted file mode 100644 index 33fd54a65..000000000 --- a/apis/README.md +++ /dev/null @@ -1,15 +0,0 @@ -## APIServer APIs - -This package aims to expose types from the plugins-sdk in the grafana apiserver. - -Currently, the types are not useable directly so we can avoid adding a dependency on k8s.io/apimachinery -until it is more necessary. See https://github.com/grafana/grafana-plugin-sdk-go/pull/909 - -The "v0alpha1" version should be considered experimental and is subject to change at any time without notice. -Once it is more stable, it will be released as a versioned API (v1) - - -### Codegen - -The file [apis/data/v0alpha1/zz_generated.deepcopy.go](data/v0alpha1/zz_generated.deepcopy.go) was generated by copying the folder structure into -https://github.com/grafana/grafana/tree/main/pkg/apis and then run `hack/update-codegen.sh data` in [hack scripts](https://github.com/grafana/grafana/tree/v10.3.3/hack). \ No newline at end of file diff --git a/apis/data/v0alpha1/client.go b/apis/data/v0alpha1/client.go deleted file mode 100644 index b84ae196a..000000000 --- a/apis/data/v0alpha1/client.go +++ /dev/null @@ -1,61 +0,0 @@ -package v0alpha1 - -import ( - "bytes" - "context" - "encoding/json" - "net/http" - - "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" -) - -type QueryDataClient interface { - QueryData(ctx context.Context, req QueryDataRequest) (int, *backend.QueryDataResponse, error) -} - -type simpleHTTPClient struct { - url string - client *http.Client - headers map[string]string -} - -func NewQueryDataClient(url string, client *http.Client, headers map[string]string) QueryDataClient { - if client == nil { - client = http.DefaultClient - } - return &simpleHTTPClient{ - url: url, - client: client, - headers: headers, - } -} - -func (c *simpleHTTPClient) QueryData(ctx context.Context, query QueryDataRequest) (int, *backend.QueryDataResponse, error) { - body, err := json.Marshal(query) - if err != nil { - return http.StatusBadRequest, nil, err - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewBuffer(body)) - if err != nil { - return http.StatusBadRequest, nil, err - } - for k, v := range c.headers { - req.Header.Set(k, v) - } - req.Header.Set("Content-Type", "application/json") - - rsp, err := c.client.Do(req) - if err != nil { - return rsp.StatusCode, nil, err - } - defer rsp.Body.Close() - - qdr := &backend.QueryDataResponse{} - iter, err := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, rsp.Body, 1024*10) - if err == nil { - err = iter.ReadVal(qdr) - } - return rsp.StatusCode, qdr, err -} diff --git a/apis/data/v0alpha1/client_test.go b/apis/data/v0alpha1/client_test.go deleted file mode 100644 index dd5eb125f..000000000 --- a/apis/data/v0alpha1/client_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package v0alpha1_test - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "testing" - - "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" - "github.com/stretchr/testify/require" -) - -func TestQueryClient(t *testing.T) { - t.Skip() - - client := v0alpha1.NewQueryDataClient("http://localhost:3000/api/ds/query", nil, - map[string]string{ - "Authorization": "Bearer XYZ", - }) - body := `{ - "from": "", - "to": "", - "queries": [ - { - "refId": "X", - "scenarioId": "csv_content", - "datasource": { - "type": "grafana-testdata-datasource", - "uid": "PD8C576611E62080A" - }, - "csvContent": "a,b,c\n1,hello,true", - "hide": true - } - ] - }` - qdr := v0alpha1.QueryDataRequest{} - err := json.Unmarshal([]byte(body), &qdr) - require.NoError(t, err) - - code, rsp, err := client.QueryData(context.Background(), qdr) - require.NoError(t, err) - require.Equal(t, http.StatusOK, code) - - r, ok := rsp.Responses["X"] - require.True(t, ok) - - for _, frame := range r.Frames { - txt, err := frame.StringTable(20, 10) - require.NoError(t, err) - fmt.Printf("%s\n", txt) - } -} diff --git a/apis/data/v0alpha1/doc.go b/apis/data/v0alpha1/doc.go deleted file mode 100644 index 6a049ca39..000000000 --- a/apis/data/v0alpha1/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// +k8s:deepcopy-gen=package -// +k8s:openapi-gen=true -// +k8s:defaulter-gen=TypeMeta -// +groupName=data.grafana.com - -package v0alpha1 diff --git a/apis/data/v0alpha1/metaV1.go b/apis/data/v0alpha1/metaV1.go deleted file mode 100644 index 042f01629..000000000 --- a/apis/data/v0alpha1/metaV1.go +++ /dev/null @@ -1,19 +0,0 @@ -package v0alpha1 - -// ObjectMeta is a struct that aims to "look" like a real kubernetes object when -// written to JSON, however it does not require the pile of dependencies -// This is really an internal helper until we decide which dependencies make sense -// to require within the SDK -type ObjectMeta struct { - // The name is for k8s and description, but not used in the schema - Name string `json:"name,omitempty"` - // Changes indicate that *something * changed - ResourceVersion string `json:"resourceVersion,omitempty"` - // Timestamp - CreationTimestamp string `json:"creationTimestamp,omitempty"` -} - -type TypeMeta struct { - Kind string `json:"kind"` // "QueryTypeDefinitionList", - APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", -} diff --git a/apis/data/v0alpha1/openapi.go b/apis/data/v0alpha1/openapi.go deleted file mode 100644 index 76366aabf..000000000 --- a/apis/data/v0alpha1/openapi.go +++ /dev/null @@ -1,82 +0,0 @@ -package v0alpha1 - -import ( - "embed" - - "k8s.io/kube-openapi/pkg/common" - spec "k8s.io/kube-openapi/pkg/validation/spec" -) - -//go:embed query.schema.json query.definition.schema.json -var f embed.FS - -func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { - return map[string]common.OpenAPIDefinition{ - "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), - "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), - "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1.DataQuery": schemaDataQuery(ref), - "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), - } -} - -// Individual response -func schemaDataResponse(_ common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "todo... improve schema", - Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{Allows: true}, - }, - }, - } -} - -func schemaDataFrame(_ common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "any object for now", - Type: []string{"object"}, - Properties: map[string]spec.Schema{}, - AdditionalProperties: &spec.SchemaOrBool{Allows: true}, - }, - }, - } -} - -func schemaQueryTypeDefinitionSpec(_ common.ReferenceCallback) common.OpenAPIDefinition { - s, _ := loadSchema("query.definition.schema.json") - if s == nil { - s = &spec.Schema{} - } - return common.OpenAPIDefinition{ - Schema: *s, - } -} - -func schemaDataQuery(_ common.ReferenceCallback) common.OpenAPIDefinition { - s, _ := DataQuerySchema() - if s == nil { - s = &spec.Schema{} - } - s.SchemaProps.Type = []string{"object"} - s.SchemaProps.AdditionalProperties = &spec.SchemaOrBool{Allows: true} - return common.OpenAPIDefinition{Schema: *s} -} - -// Get the cached feature list (exposed as a k8s resource) -func DataQuerySchema() (*spec.Schema, error) { - return loadSchema("query.schema.json") -} - -// Get the cached feature list (exposed as a k8s resource) -func loadSchema(path string) (*spec.Schema, error) { - body, err := f.ReadFile(path) - if err != nil { - return nil, err - } - s := &spec.Schema{} - err = s.UnmarshalJSON(body) - return s, err -} diff --git a/apis/data/v0alpha1/openapi_test.go b/apis/data/v0alpha1/openapi_test.go deleted file mode 100644 index 06c9eab88..000000000 --- a/apis/data/v0alpha1/openapi_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package v0alpha1 - -import ( - "encoding/json" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "k8s.io/kube-openapi/pkg/validation/spec" - "k8s.io/kube-openapi/pkg/validation/strfmt" - "k8s.io/kube-openapi/pkg/validation/validate" -) - -func TestOpenAPI(t *testing.T) { - //nolint:gocritic - defs := GetOpenAPIDefinitions(func(path string) spec.Ref { // (unlambda: replace ¯\_(ツ)_/¯) - return spec.MustCreateRef(path) // placeholder for tests - }) - - def, ok := defs["github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse"] - require.True(t, ok) - require.Empty(t, def.Dependencies) // not yet supported! - - validator := validate.NewSchemaValidator(&def.Schema, nil, "data", strfmt.Default) - - body, err := os.ReadFile("./testdata/sample_query_results.json") - require.NoError(t, err) - unstructured := make(map[string]any) - err = json.Unmarshal(body, &unstructured) - require.NoError(t, err) - - result := validator.Validate(unstructured) - for _, err := range result.Errors { - assert.NoError(t, err, "validation error") - } - for _, err := range result.Warnings { - assert.NoError(t, err, "validation warning") - } -} diff --git a/apis/data/v0alpha1/query.definition.schema.json b/apis/data/v0alpha1/query.definition.schema.json deleted file mode 100644 index d71d2569c..000000000 --- a/apis/data/v0alpha1/query.definition.schema.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-04/schema#", - "properties": { - "discriminators": { - "items": { - "properties": { - "field": { - "type": "string", - "description": "DiscriminatorField is the field used to link behavior to this specific\nquery type. It is typically \"queryType\", but can be another field if necessary" - }, - "value": { - "type": "string", - "description": "The discriminator value" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "field", - "value" - ] - }, - "type": "array", - "description": "Multiple schemas can be defined using discriminators" - }, - "description": { - "type": "string", - "description": "Describe whe the query type is for" - }, - "schema": { - "$ref": "https://json-schema.org/draft-04/schema#", - "type": "object", - "description": "The query schema represents the properties that can be sent to the API\nIn many cases, this may be the same properties that are saved in a dashboard\nIn the case where the save model is different, we must also specify a save model" - }, - "examples": { - "items": { - "properties": { - "name": { - "type": "string", - "description": "Version identifier or empty if only one exists" - }, - "description": { - "type": "string", - "description": "Optionally explain why the example is interesting" - }, - "saveModel": { - "additionalProperties": true, - "type": "object", - "description": "An example value saved that can be saved in a dashboard" - } - }, - "additionalProperties": false, - "type": "object" - }, - "type": "array", - "description": "Examples (include a wrapper) ideally a template!" - }, - "changelog": { - "items": { - "type": "string" - }, - "type": "array", - "description": "Changelog defines the changed from the previous version\nAll changes in the same version *must* be backwards compatible\nOnly notable changes will be shown here, for the full version history see git!" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "schema", - "examples" - ] -} \ No newline at end of file diff --git a/apis/data/v0alpha1/query.go b/apis/data/v0alpha1/query.go deleted file mode 100644 index a382f0ded..000000000 --- a/apis/data/v0alpha1/query.go +++ /dev/null @@ -1,419 +0,0 @@ -package v0alpha1 - -import ( - "encoding/json" - "fmt" - "unsafe" - - "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana-plugin-sdk-go/data/converters" - "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" - j "github.com/json-iterator/go" -) - -func init() { //nolint:gochecknoinits - jsoniter.RegisterTypeEncoder("v0alpha1.DataQuery", &genericQueryCodec{}) - jsoniter.RegisterTypeDecoder("v0alpha1.DataQuery", &genericQueryCodec{}) -} - -type QueryDataRequest struct { - // Time range applied to each query (when not included in the query body) - TimeRange `json:",inline"` - - // Datasource queries - Queries []DataQuery `json:"queries"` - - // Optionally include debug information in the response - Debug bool `json:"debug,omitempty"` -} - -// DataQuery is a replacement for `dtos.MetricRequest` with more explicit typing -type DataQuery struct { - CommonQueryProperties `json:",inline"` - - // Additional Properties (that live at the root) - additional map[string]any `json:"-"` // note this uses custom JSON marshalling -} - -func NewDataQuery(body map[string]any) DataQuery { - g := &DataQuery{ - additional: make(map[string]any), - } - for k, v := range body { - _ = g.Set(k, v) - } - return *g -} - -// Set allows setting values using key/value pairs -func (g *DataQuery) Set(key string, val any) *DataQuery { - switch key { - case "refId": - g.RefID, _ = val.(string) - case "resultAssertions": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.ResultAssertions) - } - case "timeRange": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.TimeRange) - } - case "datasource": - body, err := json.Marshal(val) - if err != nil { - _ = json.Unmarshal(body, &g.Datasource) - } - case "datasourceId": - v, err := converters.JSONValueToInt64.Converter(val) - if err != nil { - g.DatasourceID, _ = v.(int64) - } - case "queryType": - g.QueryType, _ = val.(string) - case "maxDataPoints": - v, err := converters.JSONValueToInt64.Converter(val) - if err != nil { - g.MaxDataPoints, _ = v.(int64) - } - case "intervalMs": - v, err := converters.JSONValueToFloat64.Converter(val) - if err != nil { - g.IntervalMS, _ = v.(float64) - } - case "hide": - g.Hide, _ = val.(bool) - default: - if g.additional == nil { - g.additional = make(map[string]any) - } - g.additional[key] = val - } - return g -} - -func (g *DataQuery) Get(key string) (any, bool) { - switch key { - case "refId": - return g.RefID, true - case "resultAssertions": - return g.ResultAssertions, true - case "timeRange": - return g.TimeRange, true - case "datasource": - return g.Datasource, true - case "datasourceId": - return g.DatasourceID, true - case "queryType": - return g.QueryType, true - case "maxDataPoints": - return g.MaxDataPoints, true - case "intervalMs": - return g.IntervalMS, true - case "hide": - return g.Hide, true - } - v, ok := g.additional[key] - return v, ok -} - -func (g *DataQuery) GetString(key string) string { - v, ok := g.Get(key) - if ok { - // At the root convert to string - s, ok := v.(string) - if ok { - return s - } - } - return "" -} - -type genericQueryCodec struct{} - -func (codec *genericQueryCodec) IsEmpty(_ unsafe.Pointer) bool { - return false -} - -func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { - q := (*DataQuery)(ptr) - writeQuery(q, stream) -} - -func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { - q := DataQuery{} - err := q.readQuery(jsoniter.NewIterator(iter)) - if err != nil { - // keep existing iter error if it exists - if iter.Error == nil { - iter.Error = err - } - return - } - *((*DataQuery)(ptr)) = q -} - -// MarshalJSON writes JSON including the common and custom values -func (g DataQuery) MarshalJSON() ([]byte, error) { - cfg := j.ConfigCompatibleWithStandardLibrary - stream := cfg.BorrowStream(nil) - defer cfg.ReturnStream(stream) - - writeQuery(&g, stream) - return append([]byte(nil), stream.Buffer()...), stream.Error -} - -// UnmarshalJSON reads a query from json byte array -func (g *DataQuery) UnmarshalJSON(b []byte) error { - iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, b) - if err != nil { - return err - } - return g.readQuery(iter) -} - -func (g *DataQuery) DeepCopyInto(out *DataQuery) { - *out = *g - g.CommonQueryProperties.DeepCopyInto(&out.CommonQueryProperties) - if g.additional != nil { - out.additional = map[string]any{} - if len(g.additional) > 0 { - jj, err := json.Marshal(g.additional) - if err != nil { - _ = json.Unmarshal(jj, &out.additional) - } - } - } -} - -func writeQuery(g *DataQuery, stream *j.Stream) { - q := g.CommonQueryProperties - stream.WriteObjectStart() - stream.WriteObjectField("refId") - stream.WriteVal(g.RefID) - - if q.ResultAssertions != nil { - stream.WriteMore() - stream.WriteObjectField("resultAssertions") - stream.WriteVal(g.ResultAssertions) - } - - if q.TimeRange != nil { - stream.WriteMore() - stream.WriteObjectField("timeRange") - stream.WriteVal(g.TimeRange) - } - - if q.Datasource != nil { - stream.WriteMore() - stream.WriteObjectField("datasource") - stream.WriteVal(g.Datasource) - } - - if q.DatasourceID > 0 { - stream.WriteMore() - stream.WriteObjectField("datasourceId") - stream.WriteVal(g.DatasourceID) - } - - if q.QueryType != "" { - stream.WriteMore() - stream.WriteObjectField("queryType") - stream.WriteVal(g.QueryType) - } - - if q.MaxDataPoints > 0 { - stream.WriteMore() - stream.WriteObjectField("maxDataPoints") - stream.WriteVal(g.MaxDataPoints) - } - - if q.IntervalMS > 0 { - stream.WriteMore() - stream.WriteObjectField("intervalMs") - stream.WriteVal(g.IntervalMS) - } - - if q.Hide { - stream.WriteMore() - stream.WriteObjectField("hide") - stream.WriteVal(g.Hide) - } - - // The additional properties - if g.additional != nil { - for k, v := range g.additional { - stream.WriteMore() - stream.WriteObjectField(k) - stream.WriteVal(v) - } - } - stream.WriteObjectEnd() -} - -func (g *DataQuery) readQuery(iter *jsoniter.Iterator) error { - return g.CommonQueryProperties.readQuery(iter, func(key string, iter *jsoniter.Iterator) error { - if g.additional == nil { - g.additional = make(map[string]any) - } - v, err := iter.Read() - g.additional[key] = v - return err - }) -} - -func (g *CommonQueryProperties) readQuery(iter *jsoniter.Iterator, - processUnknownKey func(key string, iter *jsoniter.Iterator) error, -) error { - var err error - var next j.ValueType - field := "" - for field, err = iter.ReadObject(); field != ""; field, err = iter.ReadObject() { - switch field { - case "refId": - g.RefID, err = iter.ReadString() - case "resultAssertions": - err = iter.ReadVal(&g.ResultAssertions) - case "timeRange": - err = iter.ReadVal(&g.TimeRange) - case "datasource": - // Old datasource values may just be a string - next, err = iter.WhatIsNext() - if err == nil { - switch next { - case j.StringValue: - g.Datasource = &DataSourceRef{} - g.Datasource.UID, err = iter.ReadString() - case j.ObjectValue: - err = iter.ReadVal(&g.Datasource) - default: - return fmt.Errorf("expected string or object") - } - } - - case "datasourceId": - g.DatasourceID, err = iter.ReadInt64() - case "queryType": - g.QueryType, err = iter.ReadString() - case "maxDataPoints": - g.MaxDataPoints, err = iter.ReadInt64() - case "intervalMs": - g.IntervalMS, err = iter.ReadFloat64() - case "hide": - g.Hide, err = iter.ReadBool() - default: - err = processUnknownKey(field, iter) - } - if err != nil { - return err - } - } - return err -} - -// CommonQueryProperties are properties that can be added to all queries. -// These properties live in the same JSON level as datasource specific properties, -// so care must be taken to ensure they do not overlap -type CommonQueryProperties struct { - // RefID is the unique identifier of the query, set by the frontend call. - RefID string `json:"refId,omitempty"` - - // Optionally define expected query result behavior - ResultAssertions *ResultAssertions `json:"resultAssertions,omitempty"` - - // TimeRange represents the query range - // NOTE: unlike generic /ds/query, we can now send explicit time values in each query - // NOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly - TimeRange *TimeRange `json:"timeRange,omitempty"` - - // The datasource - Datasource *DataSourceRef `json:"datasource,omitempty"` - - // Deprecated -- use datasource ref instead - DatasourceID int64 `json:"datasourceId,omitempty"` - - // QueryType is an optional identifier for the type of query. - // It can be used to distinguish different types of queries. - QueryType string `json:"queryType,omitempty"` - - // MaxDataPoints is the maximum number of data points that should be returned from a time series query. - // NOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated - // from the number of pixels visible in a visualization - MaxDataPoints int64 `json:"maxDataPoints,omitempty"` - - // Interval is the suggested duration between time points in a time series query. - // NOTE: the values for intervalMs is not saved in the query model. It is typically calculated - // from the interval required to fill a pixels in the visualization - IntervalMS float64 `json:"intervalMs,omitempty"` - - // true if query is disabled (ie should not be returned to the dashboard) - // NOTE: this does not always imply that the query should not be executed since - // the results from a hidden query may be used as the input to other queries (SSE etc) - Hide bool `json:"hide,omitempty"` -} - -type DataSourceRef struct { - // The datasource plugin type - Type string `json:"type"` - - // Datasource UID - UID string `json:"uid,omitempty"` - - // ?? the datasource API version? (just version, not the group? type | apiVersion?) -} - -// TimeRange represents a time range for a query and is a property of DataQuery. -type TimeRange struct { - // From is the start time of the query. - From string `json:"from" jsonschema:"example=now-1h,default=now-6h"` - - // To is the end time of the query. - To string `json:"to" jsonschema:"example=now,default=now"` -} - -// ResultAssertions define the expected response shape and query behavior. This is useful to -// enforce behavior over time. The assertions are passed to the query engine and can be used -// to fail queries *before* returning them to a client (select * from bigquery!) -type ResultAssertions struct { - // Type asserts that the frame matches a known type structure. - Type data.FrameType `json:"type,omitempty"` - - // TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane - // contract documentation https://grafana.github.io/dataplane/contract/. - TypeVersion data.FrameTypeVersion `json:"typeVersion"` - - // Maximum frame count - MaxFrames int64 `json:"maxFrames,omitempty"` - - // Once we can support this, adding max bytes would be helpful - // // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast - // MaxBytes int64 `json:"maxBytes,omitempty"` -} - -func (r *ResultAssertions) Validate(frames data.Frames) error { - if r.Type != data.FrameTypeUnknown { - for _, frame := range frames { - if frame.Meta == nil { - return fmt.Errorf("result missing frame type (and metadata)") - } - if frame.Meta.Type == data.FrameTypeUnknown { - // ?? should we try to detect? and see if we can use it as that type? - return fmt.Errorf("expected frame type [%s], but the type is unknown", r.Type) - } - if frame.Meta.Type != r.Type { - return fmt.Errorf("expected frame type [%s], but found [%s]", r.Type, frame.Meta.Type) - } - if !r.TypeVersion.IsZero() { - if r.TypeVersion == frame.Meta.TypeVersion { - return fmt.Errorf("type versions do not match. Expected [%s], but found [%s]", r.TypeVersion, frame.Meta.TypeVersion) - } - } - } - } - - if r.MaxFrames > 0 && len(frames) > int(r.MaxFrames) { - return fmt.Errorf("more than expected frames found") - } - return nil -} diff --git a/apis/data/v0alpha1/query.schema.json b/apis/data/v0alpha1/query.schema.json deleted file mode 100644 index 8972c3fd1..000000000 --- a/apis/data/v0alpha1/query.schema.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-04/schema#", - "properties": { - "refId": { - "type": "string", - "description": "RefID is the unique identifier of the query, set by the frontend call." - }, - "resultAssertions": { - "properties": { - "type": { - "type": "string", - "enum": [ - "", - "timeseries-wide", - "timeseries-long", - "timeseries-many", - "timeseries-multi", - "directory-listing", - "table", - "numeric-wide", - "numeric-multi", - "numeric-long", - "log-lines" - ], - "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", - "x-enum-description": {} - }, - "typeVersion": { - "items": { - "type": "integer" - }, - "type": "array", - "maxItems": 2, - "minItems": 2, - "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." - }, - "maxFrames": { - "type": "integer", - "description": "Maximum frame count" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "typeVersion" - ], - "description": "Optionally define expected query result behavior" - }, - "timeRange": { - "properties": { - "from": { - "type": "string", - "description": "From is the start time of the query.", - "default": "now-6h", - "examples": [ - "now-1h" - ] - }, - "to": { - "type": "string", - "description": "To is the end time of the query.", - "default": "now", - "examples": [ - "now" - ] - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "from", - "to" - ], - "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" - }, - "datasource": { - "properties": { - "type": { - "type": "string", - "description": "The datasource plugin type" - }, - "uid": { - "type": "string", - "description": "Datasource UID" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "type" - ], - "description": "The datasource" - }, - "queryType": { - "type": "string", - "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." - }, - "maxDataPoints": { - "type": "integer", - "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" - }, - "intervalMs": { - "type": "number", - "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" - }, - "hide": { - "type": "boolean", - "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" - } - }, - "additionalProperties": true, - "type": "object", - "description": "Generic query properties" -} \ No newline at end of file diff --git a/apis/data/v0alpha1/query_definition.go b/apis/data/v0alpha1/query_definition.go deleted file mode 100644 index 91662f7b5..000000000 --- a/apis/data/v0alpha1/query_definition.go +++ /dev/null @@ -1,78 +0,0 @@ -package v0alpha1 - -import ( - "fmt" -) - -// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition -type QueryTypeDefinition struct { - ObjectMeta `json:"metadata,omitempty"` - - Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` -} - -// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types -// For simple data sources, there may be only a single query type, however when multiple types -// exist they must be clearly specified with distinct discriminator field+value pairs -type QueryTypeDefinitionList struct { - TypeMeta `json:",inline"` - ObjectMeta `json:"metadata,omitempty"` - - Items []QueryTypeDefinition `json:"items"` -} - -type QueryTypeDefinitionSpec struct { - // Multiple schemas can be defined using discriminators - Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` - - // Describe whe the query type is for - Description string `json:"description,omitempty"` - - // The query schema represents the properties that can be sent to the API - // In many cases, this may be the same properties that are saved in a dashboard - // In the case where the save model is different, we must also specify a save model - Schema JSONSchema `json:"schema"` - - // Examples (include a wrapper) ideally a template! - Examples []QueryExample `json:"examples"` - - // Changelog defines the changed from the previous version - // All changes in the same version *must* be backwards compatible - // Only notable changes will be shown here, for the full version history see git! - Changelog []string `json:"changelog,omitempty"` -} - -type QueryExample struct { - // Version identifier or empty if only one exists - Name string `json:"name,omitempty"` - - // Optionally explain why the example is interesting - Description string `json:"description,omitempty"` - - // An example value saved that can be saved in a dashboard - SaveModel Unstructured `json:"saveModel,omitempty"` -} - -type DiscriminatorFieldValue struct { - // DiscriminatorField is the field used to link behavior to this specific - // query type. It is typically "queryType", but can be another field if necessary - Field string `json:"field"` - - // The discriminator value - Value string `json:"value"` -} - -// using any since this will often be enumerations -func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { - if len(keyvals)%2 != 0 { - panic("values must be even") - } - dis := []DiscriminatorFieldValue{} - for i := 0; i < len(keyvals); i += 2 { - dis = append(dis, DiscriminatorFieldValue{ - Field: fmt.Sprintf("%v", keyvals[i]), - Value: fmt.Sprintf("%v", keyvals[i+1]), - }) - } - return dis -} diff --git a/apis/data/v0alpha1/query_test.go b/apis/data/v0alpha1/query_test.go deleted file mode 100644 index 51ba48ab9..000000000 --- a/apis/data/v0alpha1/query_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package v0alpha1 - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestParseQueriesIntoQueryDataRequest(t *testing.T) { - request := []byte(`{ - "queries": [ - { - "refId": "A", - "datasource": { - "type": "grafana-googlesheets-datasource", - "uid": "b1808c48-9fc9-4045-82d7-081781f8a553" - }, - "cacheDurationSeconds": 300, - "spreadsheet": "spreadsheetID", - "datasourceId": 4, - "intervalMs": 30000, - "maxDataPoints": 794 - }, - { - "refId": "Z", - "datasource": "old", - "maxDataPoints": 10, - "timeRange": { - "from": "100", - "to": "200" - } - } - ], - "from": "1692624667389", - "to": "1692646267389" - }`) - - req := &QueryDataRequest{} - err := json.Unmarshal(request, req) - require.NoError(t, err) - - t.Run("verify raw unmarshal", func(t *testing.T) { - require.Len(t, req.Queries, 2) - require.Equal(t, "b1808c48-9fc9-4045-82d7-081781f8a553", req.Queries[0].Datasource.UID) - require.Equal(t, "spreadsheetID", req.Queries[0].GetString("spreadsheet")) - - // Write the query (with additional spreadsheetID) to JSON - out, err := json.MarshalIndent(req.Queries[0], "", " ") - require.NoError(t, err) - - // And read it back with standard JSON marshal functions - query := &DataQuery{} - err = json.Unmarshal(out, query) - require.NoError(t, err) - require.Equal(t, "spreadsheetID", query.GetString("spreadsheet")) - - // The second query has an explicit time range, and legacy datasource name - out, err = json.MarshalIndent(req.Queries[1], "", " ") - require.NoError(t, err) - // fmt.Printf("%s\n", string(out)) - require.JSONEq(t, `{ - "datasource": { - "type": "", ` /* NOTE! this implies legacy naming */ +` - "uid": "old" - }, - "maxDataPoints": 10, - "refId": "Z", - "timeRange": { - "from": "100", - "to": "200" - } - }`, string(out)) - }) - - t.Run("same results from either parser", func(t *testing.T) { - typed := &QueryDataRequest{} - err = json.Unmarshal(request, typed) - require.NoError(t, err) - - out1, err := json.MarshalIndent(req, "", " ") - require.NoError(t, err) - - out2, err := json.MarshalIndent(typed, "", " ") - require.NoError(t, err) - - require.JSONEq(t, string(out1), string(out2)) - }) -} - -func TestQueryBuilders(t *testing.T) { - prop := "testkey" - testQ1 := &DataQuery{} - testQ1.Set(prop, "A") - require.Equal(t, "A", testQ1.GetString(prop)) - - testQ1.Set(prop, "B") - require.Equal(t, "B", testQ1.GetString(prop)) - - testQ2 := testQ1 - testQ2.Set(prop, "C") - require.Equal(t, "C", testQ1.GetString(prop)) - require.Equal(t, "C", testQ2.GetString(prop)) - - // Uses the official field when exists - testQ2.Set("queryType", "D") - require.Equal(t, "D", testQ2.QueryType) - require.Equal(t, "D", testQ1.QueryType) - require.Equal(t, "D", testQ2.GetString("queryType")) - - // Map constructor - testQ3 := NewDataQuery(map[string]any{ - "queryType": "D", - "extra": "E", - }) - require.Equal(t, "D", testQ3.QueryType) - require.Equal(t, "E", testQ3.GetString("extra")) -} diff --git a/apis/data/v0alpha1/schema.go b/apis/data/v0alpha1/schema.go deleted file mode 100644 index a834b3364..000000000 --- a/apis/data/v0alpha1/schema.go +++ /dev/null @@ -1,70 +0,0 @@ -package v0alpha1 - -import ( - "encoding/json" - - openapi "k8s.io/kube-openapi/pkg/common" - "k8s.io/kube-openapi/pkg/validation/spec" -) - -// The k8s compatible jsonschema version -const draft04 = "https://json-schema.org/draft-04/schema#" - -type JSONSchema struct { - Spec *spec.Schema -} - -func (s JSONSchema) MarshalJSON() ([]byte, error) { - if s.Spec == nil { - return []byte("{}"), nil - } - body, err := s.Spec.MarshalJSON() - if err == nil { - // The internal format puts $schema last! - // this moves $schema first - cpy := map[string]any{} - err := json.Unmarshal(body, &cpy) - if err == nil { - return json.Marshal(cpy) - } - } - return body, err -} - -func (s *JSONSchema) UnmarshalJSON(data []byte) error { - s.Spec = &spec.Schema{} - return s.Spec.UnmarshalJSON(data) -} - -func (s JSONSchema) OpenAPIDefinition() openapi.OpenAPIDefinition { - return openapi.OpenAPIDefinition{Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: spec.MustCreateRef(draft04), - Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{Allows: true}, - }, - }} -} - -func (s *JSONSchema) DeepCopy() *JSONSchema { - if s == nil { - return nil - } - out := &JSONSchema{} - if s.Spec != nil { - out.Spec = &spec.Schema{} - jj, err := json.Marshal(s.Spec) - if err == nil { - _ = json.Unmarshal(jj, out.Spec) - } - } - return out -} - -func (s *JSONSchema) DeepCopyInto(out *JSONSchema) { - if s.Spec == nil { - out.Spec = nil - return - } - out.Spec = s.DeepCopy().Spec -} diff --git a/apis/data/v0alpha1/schema_test.go b/apis/data/v0alpha1/schema_test.go deleted file mode 100644 index ced3e0437..000000000 --- a/apis/data/v0alpha1/schema_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package v0alpha1 - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/stretchr/testify/require" - "k8s.io/kube-openapi/pkg/validation/spec" -) - -func TestSchemaSupport(t *testing.T) { - val := JSONSchema{ - Spec: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "hello", - Schema: draft04, - ID: "something", - }, - }, - } - jj, err := json.MarshalIndent(val, "", "") - require.NoError(t, err) - - fmt.Printf("%s\n", string(jj)) - - cpy := &JSONSchema{} - err = cpy.UnmarshalJSON(jj) - require.NoError(t, err) - require.Equal(t, val.Spec.Description, cpy.Spec.Description) -} diff --git a/apis/data/v0alpha1/testdata/sample_query_results.json b/apis/data/v0alpha1/testdata/sample_query_results.json deleted file mode 100644 index 4d0fd14f5..000000000 --- a/apis/data/v0alpha1/testdata/sample_query_results.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "status": 200, - "frames": [ - { - "schema": { - "refId": "A", - "meta": { - "typeVersion": [0, 0], - "custom": { - "customStat": 10 - } - }, - "fields": [ - { - "name": "time", - "type": "time", - "typeInfo": { - "frame": "time.Time", - "nullable": true - }, - "config": { - "interval": 1800000 - } - }, - { - "name": "A-series", - "type": "number", - "typeInfo": { - "frame": "float64", - "nullable": true - }, - "labels": {} - } - ] - }, - "data": { - "values": [ - [ - 1708955198367, 1708956998367, 1708958798367, 1708960598367, 1708962398367, 1708964198367, 1708965998367, - 1708967798367, 1708969598367, 1708971398367, 1708973198367, 1708974998367 - ], - [ - 8.675906980661981, 8.294773885233445, 8.273583516218238, 8.689987124182915, 9.139162216770474, - 8.822382059628058, 8.362948329273713, 8.443914703179315, 8.457037544672227, 8.17480477193586, - 7.965107052488668, 8.029678541545398 - ] - ] - } - } - ] -} diff --git a/apis/data/v0alpha1/unstructured.go b/apis/data/v0alpha1/unstructured.go deleted file mode 100644 index dbf87430c..000000000 --- a/apis/data/v0alpha1/unstructured.go +++ /dev/null @@ -1,81 +0,0 @@ -package v0alpha1 - -import ( - "encoding/json" - - openapi "k8s.io/kube-openapi/pkg/common" - spec "k8s.io/kube-openapi/pkg/validation/spec" -) - -// Unstructured allows objects that do not have Golang structs registered to be manipulated -// generically. -type Unstructured struct { - // Object is a JSON compatible map with string, float, int, bool, []interface{}, - // or map[string]interface{} children. - Object map[string]any -} - -// Create an unstructured value from any input -func AsUnstructured(v any) Unstructured { - out := Unstructured{} - body, err := json.Marshal(v) - if err == nil { - _ = json.Unmarshal(body, &out.Object) - } - return out -} - -// Produce an API definition that represents map[string]any -func (u Unstructured) OpenAPIDefinition() openapi.OpenAPIDefinition { - return openapi.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{Allows: true}, - }, - }, - } -} - -func (u *Unstructured) UnstructuredContent() map[string]interface{} { - if u.Object == nil { - return make(map[string]interface{}) - } - return u.Object -} - -func (u *Unstructured) SetUnstructuredContent(content map[string]interface{}) { - u.Object = content -} - -// MarshalJSON ensures that the unstructured object produces proper -// JSON when passed to Go's standard JSON library. -func (u Unstructured) MarshalJSON() ([]byte, error) { - return json.Marshal(u.Object) -} - -// UnmarshalJSON ensures that the unstructured object properly decodes -// JSON when passed to Go's standard JSON library. -func (u *Unstructured) UnmarshalJSON(b []byte) error { - return json.Unmarshal(b, &u.Object) -} - -func (u *Unstructured) DeepCopy() *Unstructured { - if u == nil { - return nil - } - out := new(Unstructured) - u.DeepCopyInto(out) - return out -} - -func (u *Unstructured) DeepCopyInto(out *Unstructured) { - obj := map[string]any{} - if u.Object != nil { - jj, err := json.Marshal(u.Object) - if err == nil { - _ = json.Unmarshal(jj, &obj) - } - } - out.Object = obj -} diff --git a/apis/data/v0alpha1/zz_generated.deepcopy.go b/apis/data/v0alpha1/zz_generated.deepcopy.go deleted file mode 100644 index aaf8295c5..000000000 --- a/apis/data/v0alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,187 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package v0alpha1 - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CommonQueryProperties) DeepCopyInto(out *CommonQueryProperties) { - *out = *in - if in.ResultAssertions != nil { - in, out := &in.ResultAssertions, &out.ResultAssertions - *out = new(ResultAssertions) - (*in).DeepCopyInto(*out) - } - if in.TimeRange != nil { - in, out := &in.TimeRange, &out.TimeRange - *out = new(TimeRange) - **out = **in - } - if in.Datasource != nil { - in, out := &in.Datasource, &out.Datasource - *out = new(DataSourceRef) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommonQueryProperties. -func (in *CommonQueryProperties) DeepCopy() *CommonQueryProperties { - if in == nil { - return nil - } - out := new(CommonQueryProperties) - in.DeepCopyInto(out) - return out -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataQuery. -func (in *DataQuery) DeepCopy() *DataQuery { - if in == nil { - return nil - } - out := new(DataQuery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *QueryDataRequest) DeepCopyInto(out *QueryDataRequest) { - *out = *in - out.TimeRange = in.TimeRange - if in.Queries != nil { - in, out := &in.Queries, &out.Queries - *out = make([]DataQuery, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryDataRequest. -func (in *QueryDataRequest) DeepCopy() *QueryDataRequest { - if in == nil { - return nil - } - out := new(QueryDataRequest) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DataSourceRef) DeepCopyInto(out *DataSourceRef) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataSourceRef. -func (in *DataSourceRef) DeepCopy() *DataSourceRef { - if in == nil { - return nil - } - out := new(DataSourceRef) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DiscriminatorFieldValue) DeepCopyInto(out *DiscriminatorFieldValue) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiscriminatorFieldValue. -func (in *DiscriminatorFieldValue) DeepCopy() *DiscriminatorFieldValue { - if in == nil { - return nil - } - out := new(DiscriminatorFieldValue) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *QueryExample) DeepCopyInto(out *QueryExample) { - *out = *in - in.SaveModel.DeepCopyInto(&out.SaveModel) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryExample. -func (in *QueryExample) DeepCopy() *QueryExample { - if in == nil { - return nil - } - out := new(QueryExample) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *QueryTypeDefinitionSpec) DeepCopyInto(out *QueryTypeDefinitionSpec) { - *out = *in - if in.Discriminators != nil { - in, out := &in.Discriminators, &out.Discriminators - *out = make([]DiscriminatorFieldValue, len(*in)) - copy(*out, *in) - } - in.Schema.DeepCopyInto(&out.Schema) - if in.Examples != nil { - in, out := &in.Examples, &out.Examples - *out = make([]QueryExample, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Changelog != nil { - in, out := &in.Changelog, &out.Changelog - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryTypeDefinitionSpec. -func (in *QueryTypeDefinitionSpec) DeepCopy() *QueryTypeDefinitionSpec { - if in == nil { - return nil - } - out := new(QueryTypeDefinitionSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ResultAssertions) DeepCopyInto(out *ResultAssertions) { - *out = *in - out.TypeVersion = in.TypeVersion - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResultAssertions. -func (in *ResultAssertions) DeepCopy() *ResultAssertions { - if in == nil { - return nil - } - out := new(ResultAssertions) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TimeRange) DeepCopyInto(out *TimeRange) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeRange. -func (in *TimeRange) DeepCopy() *TimeRange { - if in == nil { - return nil - } - out := new(TimeRange) - in.DeepCopyInto(out) - return out -} diff --git a/experimental/schemabuilder/example/query_test.go b/experimental/schemabuilder/example/query_test.go index edd7157c5..039c862b6 100644 --- a/experimental/schemabuilder/example/query_test.go +++ b/experimental/schemabuilder/example/query_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - data "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" + data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder" "github.com/stretchr/testify/require" ) diff --git a/experimental/schemabuilder/examples.go b/experimental/schemabuilder/examples.go index ebaffcb8d..95f9eab6e 100644 --- a/experimental/schemabuilder/examples.go +++ b/experimental/schemabuilder/examples.go @@ -1,7 +1,7 @@ package schemabuilder import ( - data "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" + data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" ) func exampleRequest(defs data.QueryTypeDefinitionList) data.QueryDataRequest { diff --git a/experimental/schemabuilder/panel.go b/experimental/schemabuilder/panel.go index 4cb591c52..e3e0a4d2f 100644 --- a/experimental/schemabuilder/panel.go +++ b/experimental/schemabuilder/panel.go @@ -1,6 +1,6 @@ package schemabuilder -import sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" +import sdkapi "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" // This is only used to write out a sample panel query // It is not public and not intended to represent a real panel diff --git a/experimental/schemabuilder/reflector.go b/experimental/schemabuilder/reflector.go index b93678fec..54d51ecf6 100644 --- a/experimental/schemabuilder/reflector.go +++ b/experimental/schemabuilder/reflector.go @@ -11,8 +11,8 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/data" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" "github.com/invopop/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/experimental/schemabuilder/reflector_test.go b/experimental/schemabuilder/reflector_test.go index 748b163f5..fd61c5b30 100644 --- a/experimental/schemabuilder/reflector_test.go +++ b/experimental/schemabuilder/reflector_test.go @@ -5,8 +5,8 @@ import ( "reflect" "testing" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" "github.com/grafana/grafana-plugin-sdk-go/data" + apisdata "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" "github.com/invopop/jsonschema" "github.com/stretchr/testify/require" ) @@ -16,8 +16,8 @@ func TestWriteQuerySchema(t *testing.T) { PluginID: []string{"dummy"}, ScanCode: []CodePaths{ { - BasePackage: "github.com/grafana/grafana-plugin-sdk-go/apis/data", - CodePath: "../../apis/data/v0alpha1", + BasePackage: "github.com/grafana/grafana-plugin-sdk-go/experimental/apis", + CodePath: "../apis/data/v0alpha1", }, { BasePackage: "github.com/grafana/grafana-plugin-sdk-go/data", @@ -30,7 +30,7 @@ func TestWriteQuerySchema(t *testing.T) { }) require.NoError(t, err) - query := builder.reflector.Reflect(&sdkapi.CommonQueryProperties{}) + query := builder.reflector.Reflect(&apisdata.CommonQueryProperties{}) updateEnumDescriptions(query) query.ID = "" query.Version = draft04 // used by kube-openapi @@ -40,24 +40,24 @@ func TestWriteQuerySchema(t *testing.T) { // // Hide this old property query.Properties.Delete("datasourceId") - outfile := "../../apis/data/v0alpha1/query.schema.json" + outfile := "../apis/data/v0alpha1/query.schema.json" old, _ := os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) // Make sure the embedded schema is loadable - schema, err := sdkapi.DataQuerySchema() + schema, err := apisdata.DataQuerySchema() require.NoError(t, err) require.Equal(t, 8, len(schema.Properties)) // Add schema for query type definition - query = builder.reflector.Reflect(&sdkapi.QueryTypeDefinitionSpec{}) + query = builder.reflector.Reflect(&apisdata.QueryTypeDefinitionSpec{}) updateEnumDescriptions(query) query.ID = "" query.Version = draft04 // used by kube-openapi - outfile = "../../apis/data/v0alpha1/query.definition.schema.json" + outfile = "../apis/data/v0alpha1/query.definition.schema.json" old, _ = os.ReadFile(outfile) maybeUpdateFile(t, outfile, query, old) - def := sdkapi.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1.QueryTypeDefinitionSpec"] + def := apisdata.GetOpenAPIDefinitions(nil)["github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1.QueryTypeDefinitionSpec"] require.Equal(t, query.Properties.Len(), len(def.Schema.Properties)) } diff --git a/experimental/schemabuilder/schema.go b/experimental/schemabuilder/schema.go index 48310d9a7..fb290bda2 100644 --- a/experimental/schemabuilder/schema.go +++ b/experimental/schemabuilder/schema.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" - sdkapi "github.com/grafana/grafana-plugin-sdk-go/apis/data/v0alpha1" + sdkapi "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" "k8s.io/kube-openapi/pkg/validation/spec" ) diff --git a/experimental/testdata/folder.golden.txt b/experimental/testdata/folder.golden.txt index 767f5a01d..aaca2da54 100644 --- a/experimental/testdata/folder.golden.txt +++ b/experimental/testdata/folder.golden.txt @@ -9,24 +9,24 @@ Frame[0] { "pathSeparator": "/" } Name: -Dimensions: 2 Fields by 20 Rows -+------------------+------------------+ -| Name: name | Name: media-type | -| Labels: | Labels: | -| Type: []string | Type: []string | -+------------------+------------------+ -| README.md | | -| actions | directory | -| authclient | directory | -| datasourcetest | directory | -| e2e | directory | -| errorsource | directory | -| featuretoggles | directory | -| fileinfo.go | | -| fileinfo_test.go | | -| ... | ... | -+------------------+------------------+ +Dimensions: 2 Fields by 21 Rows ++----------------+------------------+ +| Name: name | Name: media-type | +| Labels: | Labels: | +| Type: []string | Type: []string | ++----------------+------------------+ +| README.md | | +| actions | directory | +| apis | directory | +| authclient | directory | +| datasourcetest | directory | +| e2e | directory | +| errorsource | directory | +| featuretoggles | directory | +| fileinfo.go | | +| ... | ... | ++----------------+------------------+ ====== TEST DATA RESPONSE (arrow base64) ====== -FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAKAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAABgEAAAAAAABgAQAAAAAAAAAAAAAAAAAAYAEAAAAAAABUAAAAAAAAALgBAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAGgAAACgAAAArAAAANgAAAEQAAABPAAAAXwAAAG4AAACCAAAAnAAAALsAAADGAAAAzAAAANAAAADjAAAA8QAAAP4AAAAGAQAAAAAAAFJFQURNRS5tZGFjdGlvbnNhdXRoY2xpZW50ZGF0YXNvdXJjZXRlc3RlMmVlcnJvcnNvdXJjZWZlYXR1cmV0b2dnbGVzZmlsZWluZm8uZ29maWxlaW5mb190ZXN0LmdvZnJhbWVfc29ydGVyLmdvZnJhbWVfc29ydGVyX3Rlc3QuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlci5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyX3Rlc3QuZ29odHRwX2xvZ2dlcm1hY3Jvc21vY2tvYXV0aHRva2VucmV0cmlldmVycmVzdF9jbGllbnQuZ29zY2hlbWFidWlsZGVydGVzdGRhdGEAAAAAAAAAAAAACQAAABIAAAAbAAAAJAAAAC0AAAA2AAAANgAAADYAAAA2AAAANgAAADYAAAA2AAAAPwAAAEgAAABRAAAAWgAAAFoAAABjAAAAbAAAAAAAAABkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnkAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAANgBAAAAAAAA4AAAAAAAAAAoAgAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAuAAAAAMAAABMAAAAKAAAAAQAAADA/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAOD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAP///wgAAABQAAAARAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwidHlwZVZlcnNpb24iOlswLDBdLCJwYXRoU2VwYXJhdG9yIjoiLyJ9AAAAAAQAAABtZXRhAAAAAAIAAAB4AAAABAAAAKL///8UAAAAPAAAADwAAAAAAAAFOAAAAAEAAAAEAAAAkP///wgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA+AEAAEFSUk9XMQ== +FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAOAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABUAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYAAAAAAAAAFgAAAAAAAAACgEAAAAAAABoAQAAAAAAAAAAAAAAAAAAaAEAAAAAAABYAAAAAAAAAMABAAAAAAAAdQAAAAAAAAAAAAAAAgAAABUAAAAAAAAAAAAAAAAAAAAVAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAFAAAAB4AAAAsAAAALwAAADoAAABIAAAAUwAAAGMAAAByAAAAhgAAAKAAAAC/AAAAygAAANAAAADUAAAA5wAAAPUAAAACAQAACgEAAFJFQURNRS5tZGFjdGlvbnNhcGlzYXV0aGNsaWVudGRhdGFzb3VyY2V0ZXN0ZTJlZXJyb3Jzb3VyY2VmZWF0dXJldG9nZ2xlc2ZpbGVpbmZvLmdvZmlsZWluZm9fdGVzdC5nb2ZyYW1lX3NvcnRlci5nb2ZyYW1lX3NvcnRlcl90ZXN0LmdvZ29sZGVuX3Jlc3BvbnNlX2NoZWNrZXIuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlcl90ZXN0LmdvaHR0cF9sb2dnZXJtYWNyb3Ntb2Nrb2F1dGh0b2tlbnJldHJpZXZlcnJlc3RfY2xpZW50Lmdvc2NoZW1hYnVpbGRlcnRlc3RkYXRhAAAAAAAAAAAAAAAAAAAJAAAAEgAAABsAAAAkAAAALQAAADYAAAA/AAAAPwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAASAAAAFEAAABaAAAAYwAAAGMAAABsAAAAdQAAAGRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeQAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAAEAAEAAADYAQAAAAAAAOAAAAAAAAAAOAIAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAPgBAABBUlJPVzE= From 95b40506e0a854480d5ffb2e5bd3157cdadb3736 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 6 Mar 2024 06:45:20 -0800 Subject: [PATCH 71/71] move package --- experimental/apis/README.md | 15 + experimental/apis/data/v0alpha1/client.go | 61 +++ .../apis/data/v0alpha1/client_test.go | 53 +++ experimental/apis/data/v0alpha1/doc.go | 6 + experimental/apis/data/v0alpha1/metaV1.go | 19 + experimental/apis/data/v0alpha1/openapi.go | 82 ++++ .../apis/data/v0alpha1/openapi_test.go | 40 ++ .../v0alpha1/query.definition.schema.json | 72 +++ experimental/apis/data/v0alpha1/query.go | 419 ++++++++++++++++++ .../apis/data/v0alpha1/query.schema.json | 114 +++++ .../apis/data/v0alpha1/query_definition.go | 78 ++++ experimental/apis/data/v0alpha1/query_test.go | 118 +++++ experimental/apis/data/v0alpha1/schema.go | 70 +++ .../apis/data/v0alpha1/schema_test.go | 31 ++ .../testdata/sample_query_results.json | 51 +++ .../apis/data/v0alpha1/unstructured.go | 81 ++++ .../data/v0alpha1/zz_generated.deepcopy.go | 187 ++++++++ 17 files changed, 1497 insertions(+) create mode 100644 experimental/apis/README.md create mode 100644 experimental/apis/data/v0alpha1/client.go create mode 100644 experimental/apis/data/v0alpha1/client_test.go create mode 100644 experimental/apis/data/v0alpha1/doc.go create mode 100644 experimental/apis/data/v0alpha1/metaV1.go create mode 100644 experimental/apis/data/v0alpha1/openapi.go create mode 100644 experimental/apis/data/v0alpha1/openapi_test.go create mode 100644 experimental/apis/data/v0alpha1/query.definition.schema.json create mode 100644 experimental/apis/data/v0alpha1/query.go create mode 100644 experimental/apis/data/v0alpha1/query.schema.json create mode 100644 experimental/apis/data/v0alpha1/query_definition.go create mode 100644 experimental/apis/data/v0alpha1/query_test.go create mode 100644 experimental/apis/data/v0alpha1/schema.go create mode 100644 experimental/apis/data/v0alpha1/schema_test.go create mode 100644 experimental/apis/data/v0alpha1/testdata/sample_query_results.json create mode 100644 experimental/apis/data/v0alpha1/unstructured.go create mode 100644 experimental/apis/data/v0alpha1/zz_generated.deepcopy.go diff --git a/experimental/apis/README.md b/experimental/apis/README.md new file mode 100644 index 000000000..33fd54a65 --- /dev/null +++ b/experimental/apis/README.md @@ -0,0 +1,15 @@ +## APIServer APIs + +This package aims to expose types from the plugins-sdk in the grafana apiserver. + +Currently, the types are not useable directly so we can avoid adding a dependency on k8s.io/apimachinery +until it is more necessary. See https://github.com/grafana/grafana-plugin-sdk-go/pull/909 + +The "v0alpha1" version should be considered experimental and is subject to change at any time without notice. +Once it is more stable, it will be released as a versioned API (v1) + + +### Codegen + +The file [apis/data/v0alpha1/zz_generated.deepcopy.go](data/v0alpha1/zz_generated.deepcopy.go) was generated by copying the folder structure into +https://github.com/grafana/grafana/tree/main/pkg/apis and then run `hack/update-codegen.sh data` in [hack scripts](https://github.com/grafana/grafana/tree/v10.3.3/hack). \ No newline at end of file diff --git a/experimental/apis/data/v0alpha1/client.go b/experimental/apis/data/v0alpha1/client.go new file mode 100644 index 000000000..b84ae196a --- /dev/null +++ b/experimental/apis/data/v0alpha1/client.go @@ -0,0 +1,61 @@ +package v0alpha1 + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" +) + +type QueryDataClient interface { + QueryData(ctx context.Context, req QueryDataRequest) (int, *backend.QueryDataResponse, error) +} + +type simpleHTTPClient struct { + url string + client *http.Client + headers map[string]string +} + +func NewQueryDataClient(url string, client *http.Client, headers map[string]string) QueryDataClient { + if client == nil { + client = http.DefaultClient + } + return &simpleHTTPClient{ + url: url, + client: client, + headers: headers, + } +} + +func (c *simpleHTTPClient) QueryData(ctx context.Context, query QueryDataRequest) (int, *backend.QueryDataResponse, error) { + body, err := json.Marshal(query) + if err != nil { + return http.StatusBadRequest, nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewBuffer(body)) + if err != nil { + return http.StatusBadRequest, nil, err + } + for k, v := range c.headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", "application/json") + + rsp, err := c.client.Do(req) + if err != nil { + return rsp.StatusCode, nil, err + } + defer rsp.Body.Close() + + qdr := &backend.QueryDataResponse{} + iter, err := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, rsp.Body, 1024*10) + if err == nil { + err = iter.ReadVal(qdr) + } + return rsp.StatusCode, qdr, err +} diff --git a/experimental/apis/data/v0alpha1/client_test.go b/experimental/apis/data/v0alpha1/client_test.go new file mode 100644 index 000000000..b7915b0dd --- /dev/null +++ b/experimental/apis/data/v0alpha1/client_test.go @@ -0,0 +1,53 @@ +package v0alpha1_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" + "github.com/stretchr/testify/require" +) + +func TestQueryClient(t *testing.T) { + t.Skip() + + client := v0alpha1.NewQueryDataClient("http://localhost:3000/api/ds/query", nil, + map[string]string{ + "Authorization": "Bearer XYZ", + }) + body := `{ + "from": "", + "to": "", + "queries": [ + { + "refId": "X", + "scenarioId": "csv_content", + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "PD8C576611E62080A" + }, + "csvContent": "a,b,c\n1,hello,true", + "hide": true + } + ] + }` + qdr := v0alpha1.QueryDataRequest{} + err := json.Unmarshal([]byte(body), &qdr) + require.NoError(t, err) + + code, rsp, err := client.QueryData(context.Background(), qdr) + require.NoError(t, err) + require.Equal(t, http.StatusOK, code) + + r, ok := rsp.Responses["X"] + require.True(t, ok) + + for _, frame := range r.Frames { + txt, err := frame.StringTable(20, 10) + require.NoError(t, err) + fmt.Printf("%s\n", txt) + } +} diff --git a/experimental/apis/data/v0alpha1/doc.go b/experimental/apis/data/v0alpha1/doc.go new file mode 100644 index 000000000..6a049ca39 --- /dev/null +++ b/experimental/apis/data/v0alpha1/doc.go @@ -0,0 +1,6 @@ +// +k8s:deepcopy-gen=package +// +k8s:openapi-gen=true +// +k8s:defaulter-gen=TypeMeta +// +groupName=data.grafana.com + +package v0alpha1 diff --git a/experimental/apis/data/v0alpha1/metaV1.go b/experimental/apis/data/v0alpha1/metaV1.go new file mode 100644 index 000000000..042f01629 --- /dev/null +++ b/experimental/apis/data/v0alpha1/metaV1.go @@ -0,0 +1,19 @@ +package v0alpha1 + +// ObjectMeta is a struct that aims to "look" like a real kubernetes object when +// written to JSON, however it does not require the pile of dependencies +// This is really an internal helper until we decide which dependencies make sense +// to require within the SDK +type ObjectMeta struct { + // The name is for k8s and description, but not used in the schema + Name string `json:"name,omitempty"` + // Changes indicate that *something * changed + ResourceVersion string `json:"resourceVersion,omitempty"` + // Timestamp + CreationTimestamp string `json:"creationTimestamp,omitempty"` +} + +type TypeMeta struct { + Kind string `json:"kind"` // "QueryTypeDefinitionList", + APIVersion string `json:"apiVersion"` // "query.grafana.app/v0alpha1", +} diff --git a/experimental/apis/data/v0alpha1/openapi.go b/experimental/apis/data/v0alpha1/openapi.go new file mode 100644 index 000000000..221ec70a6 --- /dev/null +++ b/experimental/apis/data/v0alpha1/openapi.go @@ -0,0 +1,82 @@ +package v0alpha1 + +import ( + "embed" + + "k8s.io/kube-openapi/pkg/common" + spec "k8s.io/kube-openapi/pkg/validation/spec" +) + +//go:embed query.schema.json query.definition.schema.json +var f embed.FS + +func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + return map[string]common.OpenAPIDefinition{ + "github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse": schemaDataResponse(ref), + "github.com/grafana/grafana-plugin-sdk-go/data.Frame": schemaDataFrame(ref), + "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1.DataQuery": schemaDataQuery(ref), + "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1.QueryTypeDefinitionSpec": schemaQueryTypeDefinitionSpec(ref), + } +} + +// Individual response +func schemaDataResponse(_ common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "todo... improve schema", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + }, + } +} + +func schemaDataFrame(_ common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "any object for now", + Type: []string{"object"}, + Properties: map[string]spec.Schema{}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + }, + } +} + +func schemaQueryTypeDefinitionSpec(_ common.ReferenceCallback) common.OpenAPIDefinition { + s, _ := loadSchema("query.definition.schema.json") + if s == nil { + s = &spec.Schema{} + } + return common.OpenAPIDefinition{ + Schema: *s, + } +} + +func schemaDataQuery(_ common.ReferenceCallback) common.OpenAPIDefinition { + s, _ := DataQuerySchema() + if s == nil { + s = &spec.Schema{} + } + s.SchemaProps.Type = []string{"object"} + s.SchemaProps.AdditionalProperties = &spec.SchemaOrBool{Allows: true} + return common.OpenAPIDefinition{Schema: *s} +} + +// Get the cached feature list (exposed as a k8s resource) +func DataQuerySchema() (*spec.Schema, error) { + return loadSchema("query.schema.json") +} + +// Get the cached feature list (exposed as a k8s resource) +func loadSchema(path string) (*spec.Schema, error) { + body, err := f.ReadFile(path) + if err != nil { + return nil, err + } + s := &spec.Schema{} + err = s.UnmarshalJSON(body) + return s, err +} diff --git a/experimental/apis/data/v0alpha1/openapi_test.go b/experimental/apis/data/v0alpha1/openapi_test.go new file mode 100644 index 000000000..06c9eab88 --- /dev/null +++ b/experimental/apis/data/v0alpha1/openapi_test.go @@ -0,0 +1,40 @@ +package v0alpha1 + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/kube-openapi/pkg/validation/spec" + "k8s.io/kube-openapi/pkg/validation/strfmt" + "k8s.io/kube-openapi/pkg/validation/validate" +) + +func TestOpenAPI(t *testing.T) { + //nolint:gocritic + defs := GetOpenAPIDefinitions(func(path string) spec.Ref { // (unlambda: replace ¯\_(ツ)_/¯) + return spec.MustCreateRef(path) // placeholder for tests + }) + + def, ok := defs["github.com/grafana/grafana-plugin-sdk-go/backend.DataResponse"] + require.True(t, ok) + require.Empty(t, def.Dependencies) // not yet supported! + + validator := validate.NewSchemaValidator(&def.Schema, nil, "data", strfmt.Default) + + body, err := os.ReadFile("./testdata/sample_query_results.json") + require.NoError(t, err) + unstructured := make(map[string]any) + err = json.Unmarshal(body, &unstructured) + require.NoError(t, err) + + result := validator.Validate(unstructured) + for _, err := range result.Errors { + assert.NoError(t, err, "validation error") + } + for _, err := range result.Warnings { + assert.NoError(t, err, "validation warning") + } +} diff --git a/experimental/apis/data/v0alpha1/query.definition.schema.json b/experimental/apis/data/v0alpha1/query.definition.schema.json new file mode 100644 index 000000000..d71d2569c --- /dev/null +++ b/experimental/apis/data/v0alpha1/query.definition.schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema#", + "properties": { + "discriminators": { + "items": { + "properties": { + "field": { + "type": "string", + "description": "DiscriminatorField is the field used to link behavior to this specific\nquery type. It is typically \"queryType\", but can be another field if necessary" + }, + "value": { + "type": "string", + "description": "The discriminator value" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "field", + "value" + ] + }, + "type": "array", + "description": "Multiple schemas can be defined using discriminators" + }, + "description": { + "type": "string", + "description": "Describe whe the query type is for" + }, + "schema": { + "$ref": "https://json-schema.org/draft-04/schema#", + "type": "object", + "description": "The query schema represents the properties that can be sent to the API\nIn many cases, this may be the same properties that are saved in a dashboard\nIn the case where the save model is different, we must also specify a save model" + }, + "examples": { + "items": { + "properties": { + "name": { + "type": "string", + "description": "Version identifier or empty if only one exists" + }, + "description": { + "type": "string", + "description": "Optionally explain why the example is interesting" + }, + "saveModel": { + "additionalProperties": true, + "type": "object", + "description": "An example value saved that can be saved in a dashboard" + } + }, + "additionalProperties": false, + "type": "object" + }, + "type": "array", + "description": "Examples (include a wrapper) ideally a template!" + }, + "changelog": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Changelog defines the changed from the previous version\nAll changes in the same version *must* be backwards compatible\nOnly notable changes will be shown here, for the full version history see git!" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "schema", + "examples" + ] +} \ No newline at end of file diff --git a/experimental/apis/data/v0alpha1/query.go b/experimental/apis/data/v0alpha1/query.go new file mode 100644 index 000000000..a382f0ded --- /dev/null +++ b/experimental/apis/data/v0alpha1/query.go @@ -0,0 +1,419 @@ +package v0alpha1 + +import ( + "encoding/json" + "fmt" + "unsafe" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana-plugin-sdk-go/data/converters" + "github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" + j "github.com/json-iterator/go" +) + +func init() { //nolint:gochecknoinits + jsoniter.RegisterTypeEncoder("v0alpha1.DataQuery", &genericQueryCodec{}) + jsoniter.RegisterTypeDecoder("v0alpha1.DataQuery", &genericQueryCodec{}) +} + +type QueryDataRequest struct { + // Time range applied to each query (when not included in the query body) + TimeRange `json:",inline"` + + // Datasource queries + Queries []DataQuery `json:"queries"` + + // Optionally include debug information in the response + Debug bool `json:"debug,omitempty"` +} + +// DataQuery is a replacement for `dtos.MetricRequest` with more explicit typing +type DataQuery struct { + CommonQueryProperties `json:",inline"` + + // Additional Properties (that live at the root) + additional map[string]any `json:"-"` // note this uses custom JSON marshalling +} + +func NewDataQuery(body map[string]any) DataQuery { + g := &DataQuery{ + additional: make(map[string]any), + } + for k, v := range body { + _ = g.Set(k, v) + } + return *g +} + +// Set allows setting values using key/value pairs +func (g *DataQuery) Set(key string, val any) *DataQuery { + switch key { + case "refId": + g.RefID, _ = val.(string) + case "resultAssertions": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.ResultAssertions) + } + case "timeRange": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.TimeRange) + } + case "datasource": + body, err := json.Marshal(val) + if err != nil { + _ = json.Unmarshal(body, &g.Datasource) + } + case "datasourceId": + v, err := converters.JSONValueToInt64.Converter(val) + if err != nil { + g.DatasourceID, _ = v.(int64) + } + case "queryType": + g.QueryType, _ = val.(string) + case "maxDataPoints": + v, err := converters.JSONValueToInt64.Converter(val) + if err != nil { + g.MaxDataPoints, _ = v.(int64) + } + case "intervalMs": + v, err := converters.JSONValueToFloat64.Converter(val) + if err != nil { + g.IntervalMS, _ = v.(float64) + } + case "hide": + g.Hide, _ = val.(bool) + default: + if g.additional == nil { + g.additional = make(map[string]any) + } + g.additional[key] = val + } + return g +} + +func (g *DataQuery) Get(key string) (any, bool) { + switch key { + case "refId": + return g.RefID, true + case "resultAssertions": + return g.ResultAssertions, true + case "timeRange": + return g.TimeRange, true + case "datasource": + return g.Datasource, true + case "datasourceId": + return g.DatasourceID, true + case "queryType": + return g.QueryType, true + case "maxDataPoints": + return g.MaxDataPoints, true + case "intervalMs": + return g.IntervalMS, true + case "hide": + return g.Hide, true + } + v, ok := g.additional[key] + return v, ok +} + +func (g *DataQuery) GetString(key string) string { + v, ok := g.Get(key) + if ok { + // At the root convert to string + s, ok := v.(string) + if ok { + return s + } + } + return "" +} + +type genericQueryCodec struct{} + +func (codec *genericQueryCodec) IsEmpty(_ unsafe.Pointer) bool { + return false +} + +func (codec *genericQueryCodec) Encode(ptr unsafe.Pointer, stream *j.Stream) { + q := (*DataQuery)(ptr) + writeQuery(q, stream) +} + +func (codec *genericQueryCodec) Decode(ptr unsafe.Pointer, iter *j.Iterator) { + q := DataQuery{} + err := q.readQuery(jsoniter.NewIterator(iter)) + if err != nil { + // keep existing iter error if it exists + if iter.Error == nil { + iter.Error = err + } + return + } + *((*DataQuery)(ptr)) = q +} + +// MarshalJSON writes JSON including the common and custom values +func (g DataQuery) MarshalJSON() ([]byte, error) { + cfg := j.ConfigCompatibleWithStandardLibrary + stream := cfg.BorrowStream(nil) + defer cfg.ReturnStream(stream) + + writeQuery(&g, stream) + return append([]byte(nil), stream.Buffer()...), stream.Error +} + +// UnmarshalJSON reads a query from json byte array +func (g *DataQuery) UnmarshalJSON(b []byte) error { + iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, b) + if err != nil { + return err + } + return g.readQuery(iter) +} + +func (g *DataQuery) DeepCopyInto(out *DataQuery) { + *out = *g + g.CommonQueryProperties.DeepCopyInto(&out.CommonQueryProperties) + if g.additional != nil { + out.additional = map[string]any{} + if len(g.additional) > 0 { + jj, err := json.Marshal(g.additional) + if err != nil { + _ = json.Unmarshal(jj, &out.additional) + } + } + } +} + +func writeQuery(g *DataQuery, stream *j.Stream) { + q := g.CommonQueryProperties + stream.WriteObjectStart() + stream.WriteObjectField("refId") + stream.WriteVal(g.RefID) + + if q.ResultAssertions != nil { + stream.WriteMore() + stream.WriteObjectField("resultAssertions") + stream.WriteVal(g.ResultAssertions) + } + + if q.TimeRange != nil { + stream.WriteMore() + stream.WriteObjectField("timeRange") + stream.WriteVal(g.TimeRange) + } + + if q.Datasource != nil { + stream.WriteMore() + stream.WriteObjectField("datasource") + stream.WriteVal(g.Datasource) + } + + if q.DatasourceID > 0 { + stream.WriteMore() + stream.WriteObjectField("datasourceId") + stream.WriteVal(g.DatasourceID) + } + + if q.QueryType != "" { + stream.WriteMore() + stream.WriteObjectField("queryType") + stream.WriteVal(g.QueryType) + } + + if q.MaxDataPoints > 0 { + stream.WriteMore() + stream.WriteObjectField("maxDataPoints") + stream.WriteVal(g.MaxDataPoints) + } + + if q.IntervalMS > 0 { + stream.WriteMore() + stream.WriteObjectField("intervalMs") + stream.WriteVal(g.IntervalMS) + } + + if q.Hide { + stream.WriteMore() + stream.WriteObjectField("hide") + stream.WriteVal(g.Hide) + } + + // The additional properties + if g.additional != nil { + for k, v := range g.additional { + stream.WriteMore() + stream.WriteObjectField(k) + stream.WriteVal(v) + } + } + stream.WriteObjectEnd() +} + +func (g *DataQuery) readQuery(iter *jsoniter.Iterator) error { + return g.CommonQueryProperties.readQuery(iter, func(key string, iter *jsoniter.Iterator) error { + if g.additional == nil { + g.additional = make(map[string]any) + } + v, err := iter.Read() + g.additional[key] = v + return err + }) +} + +func (g *CommonQueryProperties) readQuery(iter *jsoniter.Iterator, + processUnknownKey func(key string, iter *jsoniter.Iterator) error, +) error { + var err error + var next j.ValueType + field := "" + for field, err = iter.ReadObject(); field != ""; field, err = iter.ReadObject() { + switch field { + case "refId": + g.RefID, err = iter.ReadString() + case "resultAssertions": + err = iter.ReadVal(&g.ResultAssertions) + case "timeRange": + err = iter.ReadVal(&g.TimeRange) + case "datasource": + // Old datasource values may just be a string + next, err = iter.WhatIsNext() + if err == nil { + switch next { + case j.StringValue: + g.Datasource = &DataSourceRef{} + g.Datasource.UID, err = iter.ReadString() + case j.ObjectValue: + err = iter.ReadVal(&g.Datasource) + default: + return fmt.Errorf("expected string or object") + } + } + + case "datasourceId": + g.DatasourceID, err = iter.ReadInt64() + case "queryType": + g.QueryType, err = iter.ReadString() + case "maxDataPoints": + g.MaxDataPoints, err = iter.ReadInt64() + case "intervalMs": + g.IntervalMS, err = iter.ReadFloat64() + case "hide": + g.Hide, err = iter.ReadBool() + default: + err = processUnknownKey(field, iter) + } + if err != nil { + return err + } + } + return err +} + +// CommonQueryProperties are properties that can be added to all queries. +// These properties live in the same JSON level as datasource specific properties, +// so care must be taken to ensure they do not overlap +type CommonQueryProperties struct { + // RefID is the unique identifier of the query, set by the frontend call. + RefID string `json:"refId,omitempty"` + + // Optionally define expected query result behavior + ResultAssertions *ResultAssertions `json:"resultAssertions,omitempty"` + + // TimeRange represents the query range + // NOTE: unlike generic /ds/query, we can now send explicit time values in each query + // NOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly + TimeRange *TimeRange `json:"timeRange,omitempty"` + + // The datasource + Datasource *DataSourceRef `json:"datasource,omitempty"` + + // Deprecated -- use datasource ref instead + DatasourceID int64 `json:"datasourceId,omitempty"` + + // QueryType is an optional identifier for the type of query. + // It can be used to distinguish different types of queries. + QueryType string `json:"queryType,omitempty"` + + // MaxDataPoints is the maximum number of data points that should be returned from a time series query. + // NOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated + // from the number of pixels visible in a visualization + MaxDataPoints int64 `json:"maxDataPoints,omitempty"` + + // Interval is the suggested duration between time points in a time series query. + // NOTE: the values for intervalMs is not saved in the query model. It is typically calculated + // from the interval required to fill a pixels in the visualization + IntervalMS float64 `json:"intervalMs,omitempty"` + + // true if query is disabled (ie should not be returned to the dashboard) + // NOTE: this does not always imply that the query should not be executed since + // the results from a hidden query may be used as the input to other queries (SSE etc) + Hide bool `json:"hide,omitempty"` +} + +type DataSourceRef struct { + // The datasource plugin type + Type string `json:"type"` + + // Datasource UID + UID string `json:"uid,omitempty"` + + // ?? the datasource API version? (just version, not the group? type | apiVersion?) +} + +// TimeRange represents a time range for a query and is a property of DataQuery. +type TimeRange struct { + // From is the start time of the query. + From string `json:"from" jsonschema:"example=now-1h,default=now-6h"` + + // To is the end time of the query. + To string `json:"to" jsonschema:"example=now,default=now"` +} + +// ResultAssertions define the expected response shape and query behavior. This is useful to +// enforce behavior over time. The assertions are passed to the query engine and can be used +// to fail queries *before* returning them to a client (select * from bigquery!) +type ResultAssertions struct { + // Type asserts that the frame matches a known type structure. + Type data.FrameType `json:"type,omitempty"` + + // TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane + // contract documentation https://grafana.github.io/dataplane/contract/. + TypeVersion data.FrameTypeVersion `json:"typeVersion"` + + // Maximum frame count + MaxFrames int64 `json:"maxFrames,omitempty"` + + // Once we can support this, adding max bytes would be helpful + // // Maximum bytes that can be read -- if the query planning expects more then this, the query may fail fast + // MaxBytes int64 `json:"maxBytes,omitempty"` +} + +func (r *ResultAssertions) Validate(frames data.Frames) error { + if r.Type != data.FrameTypeUnknown { + for _, frame := range frames { + if frame.Meta == nil { + return fmt.Errorf("result missing frame type (and metadata)") + } + if frame.Meta.Type == data.FrameTypeUnknown { + // ?? should we try to detect? and see if we can use it as that type? + return fmt.Errorf("expected frame type [%s], but the type is unknown", r.Type) + } + if frame.Meta.Type != r.Type { + return fmt.Errorf("expected frame type [%s], but found [%s]", r.Type, frame.Meta.Type) + } + if !r.TypeVersion.IsZero() { + if r.TypeVersion == frame.Meta.TypeVersion { + return fmt.Errorf("type versions do not match. Expected [%s], but found [%s]", r.TypeVersion, frame.Meta.TypeVersion) + } + } + } + } + + if r.MaxFrames > 0 && len(frames) > int(r.MaxFrames) { + return fmt.Errorf("more than expected frames found") + } + return nil +} diff --git a/experimental/apis/data/v0alpha1/query.schema.json b/experimental/apis/data/v0alpha1/query.schema.json new file mode 100644 index 000000000..8972c3fd1 --- /dev/null +++ b/experimental/apis/data/v0alpha1/query.schema.json @@ -0,0 +1,114 @@ +{ + "$schema": "https://json-schema.org/draft-04/schema#", + "properties": { + "refId": { + "type": "string", + "description": "RefID is the unique identifier of the query, set by the frontend call." + }, + "resultAssertions": { + "properties": { + "type": { + "type": "string", + "enum": [ + "", + "timeseries-wide", + "timeseries-long", + "timeseries-many", + "timeseries-multi", + "directory-listing", + "table", + "numeric-wide", + "numeric-multi", + "numeric-long", + "log-lines" + ], + "description": "Type asserts that the frame matches a known type structure.\n\n\nPossible enum values:\n - `\"\"` \n - `\"timeseries-wide\"` \n - `\"timeseries-long\"` \n - `\"timeseries-many\"` \n - `\"timeseries-multi\"` \n - `\"directory-listing\"` \n - `\"table\"` \n - `\"numeric-wide\"` \n - `\"numeric-multi\"` \n - `\"numeric-long\"` \n - `\"log-lines\"` ", + "x-enum-description": {} + }, + "typeVersion": { + "items": { + "type": "integer" + }, + "type": "array", + "maxItems": 2, + "minItems": 2, + "description": "TypeVersion is the version of the Type property. Versions greater than 0.0 correspond to the dataplane\ncontract documentation https://grafana.github.io/dataplane/contract/." + }, + "maxFrames": { + "type": "integer", + "description": "Maximum frame count" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "typeVersion" + ], + "description": "Optionally define expected query result behavior" + }, + "timeRange": { + "properties": { + "from": { + "type": "string", + "description": "From is the start time of the query.", + "default": "now-6h", + "examples": [ + "now-1h" + ] + }, + "to": { + "type": "string", + "description": "To is the end time of the query.", + "default": "now", + "examples": [ + "now" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "from", + "to" + ], + "description": "TimeRange represents the query range\nNOTE: unlike generic /ds/query, we can now send explicit time values in each query\nNOTE: the values for timeRange are not saved in a dashboard, they are constructed on the fly" + }, + "datasource": { + "properties": { + "type": { + "type": "string", + "description": "The datasource plugin type" + }, + "uid": { + "type": "string", + "description": "Datasource UID" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ], + "description": "The datasource" + }, + "queryType": { + "type": "string", + "description": "QueryType is an optional identifier for the type of query.\nIt can be used to distinguish different types of queries." + }, + "maxDataPoints": { + "type": "integer", + "description": "MaxDataPoints is the maximum number of data points that should be returned from a time series query.\nNOTE: the values for maxDataPoints is not saved in the query model. It is typically calculated\nfrom the number of pixels visible in a visualization" + }, + "intervalMs": { + "type": "number", + "description": "Interval is the suggested duration between time points in a time series query.\nNOTE: the values for intervalMs is not saved in the query model. It is typically calculated\nfrom the interval required to fill a pixels in the visualization" + }, + "hide": { + "type": "boolean", + "description": "true if query is disabled (ie should not be returned to the dashboard)\nNOTE: this does not always imply that the query should not be executed since\nthe results from a hidden query may be used as the input to other queries (SSE etc)" + } + }, + "additionalProperties": true, + "type": "object", + "description": "Generic query properties" +} \ No newline at end of file diff --git a/experimental/apis/data/v0alpha1/query_definition.go b/experimental/apis/data/v0alpha1/query_definition.go new file mode 100644 index 000000000..91662f7b5 --- /dev/null +++ b/experimental/apis/data/v0alpha1/query_definition.go @@ -0,0 +1,78 @@ +package v0alpha1 + +import ( + "fmt" +) + +// QueryTypeDefinition is a kubernetes shaped object that represents a single query definition +type QueryTypeDefinition struct { + ObjectMeta `json:"metadata,omitempty"` + + Spec QueryTypeDefinitionSpec `json:"spec,omitempty"` +} + +// QueryTypeDefinitionList is a kubernetes shaped object that represents a list of query types +// For simple data sources, there may be only a single query type, however when multiple types +// exist they must be clearly specified with distinct discriminator field+value pairs +type QueryTypeDefinitionList struct { + TypeMeta `json:",inline"` + ObjectMeta `json:"metadata,omitempty"` + + Items []QueryTypeDefinition `json:"items"` +} + +type QueryTypeDefinitionSpec struct { + // Multiple schemas can be defined using discriminators + Discriminators []DiscriminatorFieldValue `json:"discriminators,omitempty"` + + // Describe whe the query type is for + Description string `json:"description,omitempty"` + + // The query schema represents the properties that can be sent to the API + // In many cases, this may be the same properties that are saved in a dashboard + // In the case where the save model is different, we must also specify a save model + Schema JSONSchema `json:"schema"` + + // Examples (include a wrapper) ideally a template! + Examples []QueryExample `json:"examples"` + + // Changelog defines the changed from the previous version + // All changes in the same version *must* be backwards compatible + // Only notable changes will be shown here, for the full version history see git! + Changelog []string `json:"changelog,omitempty"` +} + +type QueryExample struct { + // Version identifier or empty if only one exists + Name string `json:"name,omitempty"` + + // Optionally explain why the example is interesting + Description string `json:"description,omitempty"` + + // An example value saved that can be saved in a dashboard + SaveModel Unstructured `json:"saveModel,omitempty"` +} + +type DiscriminatorFieldValue struct { + // DiscriminatorField is the field used to link behavior to this specific + // query type. It is typically "queryType", but can be another field if necessary + Field string `json:"field"` + + // The discriminator value + Value string `json:"value"` +} + +// using any since this will often be enumerations +func NewDiscriminators(keyvals ...any) []DiscriminatorFieldValue { + if len(keyvals)%2 != 0 { + panic("values must be even") + } + dis := []DiscriminatorFieldValue{} + for i := 0; i < len(keyvals); i += 2 { + dis = append(dis, DiscriminatorFieldValue{ + Field: fmt.Sprintf("%v", keyvals[i]), + Value: fmt.Sprintf("%v", keyvals[i+1]), + }) + } + return dis +} diff --git a/experimental/apis/data/v0alpha1/query_test.go b/experimental/apis/data/v0alpha1/query_test.go new file mode 100644 index 000000000..51ba48ab9 --- /dev/null +++ b/experimental/apis/data/v0alpha1/query_test.go @@ -0,0 +1,118 @@ +package v0alpha1 + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseQueriesIntoQueryDataRequest(t *testing.T) { + request := []byte(`{ + "queries": [ + { + "refId": "A", + "datasource": { + "type": "grafana-googlesheets-datasource", + "uid": "b1808c48-9fc9-4045-82d7-081781f8a553" + }, + "cacheDurationSeconds": 300, + "spreadsheet": "spreadsheetID", + "datasourceId": 4, + "intervalMs": 30000, + "maxDataPoints": 794 + }, + { + "refId": "Z", + "datasource": "old", + "maxDataPoints": 10, + "timeRange": { + "from": "100", + "to": "200" + } + } + ], + "from": "1692624667389", + "to": "1692646267389" + }`) + + req := &QueryDataRequest{} + err := json.Unmarshal(request, req) + require.NoError(t, err) + + t.Run("verify raw unmarshal", func(t *testing.T) { + require.Len(t, req.Queries, 2) + require.Equal(t, "b1808c48-9fc9-4045-82d7-081781f8a553", req.Queries[0].Datasource.UID) + require.Equal(t, "spreadsheetID", req.Queries[0].GetString("spreadsheet")) + + // Write the query (with additional spreadsheetID) to JSON + out, err := json.MarshalIndent(req.Queries[0], "", " ") + require.NoError(t, err) + + // And read it back with standard JSON marshal functions + query := &DataQuery{} + err = json.Unmarshal(out, query) + require.NoError(t, err) + require.Equal(t, "spreadsheetID", query.GetString("spreadsheet")) + + // The second query has an explicit time range, and legacy datasource name + out, err = json.MarshalIndent(req.Queries[1], "", " ") + require.NoError(t, err) + // fmt.Printf("%s\n", string(out)) + require.JSONEq(t, `{ + "datasource": { + "type": "", ` /* NOTE! this implies legacy naming */ +` + "uid": "old" + }, + "maxDataPoints": 10, + "refId": "Z", + "timeRange": { + "from": "100", + "to": "200" + } + }`, string(out)) + }) + + t.Run("same results from either parser", func(t *testing.T) { + typed := &QueryDataRequest{} + err = json.Unmarshal(request, typed) + require.NoError(t, err) + + out1, err := json.MarshalIndent(req, "", " ") + require.NoError(t, err) + + out2, err := json.MarshalIndent(typed, "", " ") + require.NoError(t, err) + + require.JSONEq(t, string(out1), string(out2)) + }) +} + +func TestQueryBuilders(t *testing.T) { + prop := "testkey" + testQ1 := &DataQuery{} + testQ1.Set(prop, "A") + require.Equal(t, "A", testQ1.GetString(prop)) + + testQ1.Set(prop, "B") + require.Equal(t, "B", testQ1.GetString(prop)) + + testQ2 := testQ1 + testQ2.Set(prop, "C") + require.Equal(t, "C", testQ1.GetString(prop)) + require.Equal(t, "C", testQ2.GetString(prop)) + + // Uses the official field when exists + testQ2.Set("queryType", "D") + require.Equal(t, "D", testQ2.QueryType) + require.Equal(t, "D", testQ1.QueryType) + require.Equal(t, "D", testQ2.GetString("queryType")) + + // Map constructor + testQ3 := NewDataQuery(map[string]any{ + "queryType": "D", + "extra": "E", + }) + require.Equal(t, "D", testQ3.QueryType) + require.Equal(t, "E", testQ3.GetString("extra")) +} diff --git a/experimental/apis/data/v0alpha1/schema.go b/experimental/apis/data/v0alpha1/schema.go new file mode 100644 index 000000000..a834b3364 --- /dev/null +++ b/experimental/apis/data/v0alpha1/schema.go @@ -0,0 +1,70 @@ +package v0alpha1 + +import ( + "encoding/json" + + openapi "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +// The k8s compatible jsonschema version +const draft04 = "https://json-schema.org/draft-04/schema#" + +type JSONSchema struct { + Spec *spec.Schema +} + +func (s JSONSchema) MarshalJSON() ([]byte, error) { + if s.Spec == nil { + return []byte("{}"), nil + } + body, err := s.Spec.MarshalJSON() + if err == nil { + // The internal format puts $schema last! + // this moves $schema first + cpy := map[string]any{} + err := json.Unmarshal(body, &cpy) + if err == nil { + return json.Marshal(cpy) + } + } + return body, err +} + +func (s *JSONSchema) UnmarshalJSON(data []byte) error { + s.Spec = &spec.Schema{} + return s.Spec.UnmarshalJSON(data) +} + +func (s JSONSchema) OpenAPIDefinition() openapi.OpenAPIDefinition { + return openapi.OpenAPIDefinition{Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: spec.MustCreateRef(draft04), + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + }} +} + +func (s *JSONSchema) DeepCopy() *JSONSchema { + if s == nil { + return nil + } + out := &JSONSchema{} + if s.Spec != nil { + out.Spec = &spec.Schema{} + jj, err := json.Marshal(s.Spec) + if err == nil { + _ = json.Unmarshal(jj, out.Spec) + } + } + return out +} + +func (s *JSONSchema) DeepCopyInto(out *JSONSchema) { + if s.Spec == nil { + out.Spec = nil + return + } + out.Spec = s.DeepCopy().Spec +} diff --git a/experimental/apis/data/v0alpha1/schema_test.go b/experimental/apis/data/v0alpha1/schema_test.go new file mode 100644 index 000000000..ced3e0437 --- /dev/null +++ b/experimental/apis/data/v0alpha1/schema_test.go @@ -0,0 +1,31 @@ +package v0alpha1 + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +func TestSchemaSupport(t *testing.T) { + val := JSONSchema{ + Spec: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "hello", + Schema: draft04, + ID: "something", + }, + }, + } + jj, err := json.MarshalIndent(val, "", "") + require.NoError(t, err) + + fmt.Printf("%s\n", string(jj)) + + cpy := &JSONSchema{} + err = cpy.UnmarshalJSON(jj) + require.NoError(t, err) + require.Equal(t, val.Spec.Description, cpy.Spec.Description) +} diff --git a/experimental/apis/data/v0alpha1/testdata/sample_query_results.json b/experimental/apis/data/v0alpha1/testdata/sample_query_results.json new file mode 100644 index 000000000..4d0fd14f5 --- /dev/null +++ b/experimental/apis/data/v0alpha1/testdata/sample_query_results.json @@ -0,0 +1,51 @@ +{ + "status": 200, + "frames": [ + { + "schema": { + "refId": "A", + "meta": { + "typeVersion": [0, 0], + "custom": { + "customStat": 10 + } + }, + "fields": [ + { + "name": "time", + "type": "time", + "typeInfo": { + "frame": "time.Time", + "nullable": true + }, + "config": { + "interval": 1800000 + } + }, + { + "name": "A-series", + "type": "number", + "typeInfo": { + "frame": "float64", + "nullable": true + }, + "labels": {} + } + ] + }, + "data": { + "values": [ + [ + 1708955198367, 1708956998367, 1708958798367, 1708960598367, 1708962398367, 1708964198367, 1708965998367, + 1708967798367, 1708969598367, 1708971398367, 1708973198367, 1708974998367 + ], + [ + 8.675906980661981, 8.294773885233445, 8.273583516218238, 8.689987124182915, 9.139162216770474, + 8.822382059628058, 8.362948329273713, 8.443914703179315, 8.457037544672227, 8.17480477193586, + 7.965107052488668, 8.029678541545398 + ] + ] + } + } + ] +} diff --git a/experimental/apis/data/v0alpha1/unstructured.go b/experimental/apis/data/v0alpha1/unstructured.go new file mode 100644 index 000000000..dbf87430c --- /dev/null +++ b/experimental/apis/data/v0alpha1/unstructured.go @@ -0,0 +1,81 @@ +package v0alpha1 + +import ( + "encoding/json" + + openapi "k8s.io/kube-openapi/pkg/common" + spec "k8s.io/kube-openapi/pkg/validation/spec" +) + +// Unstructured allows objects that do not have Golang structs registered to be manipulated +// generically. +type Unstructured struct { + // Object is a JSON compatible map with string, float, int, bool, []interface{}, + // or map[string]interface{} children. + Object map[string]any +} + +// Create an unstructured value from any input +func AsUnstructured(v any) Unstructured { + out := Unstructured{} + body, err := json.Marshal(v) + if err == nil { + _ = json.Unmarshal(body, &out.Object) + } + return out +} + +// Produce an API definition that represents map[string]any +func (u Unstructured) OpenAPIDefinition() openapi.OpenAPIDefinition { + return openapi.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + }, + } +} + +func (u *Unstructured) UnstructuredContent() map[string]interface{} { + if u.Object == nil { + return make(map[string]interface{}) + } + return u.Object +} + +func (u *Unstructured) SetUnstructuredContent(content map[string]interface{}) { + u.Object = content +} + +// MarshalJSON ensures that the unstructured object produces proper +// JSON when passed to Go's standard JSON library. +func (u Unstructured) MarshalJSON() ([]byte, error) { + return json.Marshal(u.Object) +} + +// UnmarshalJSON ensures that the unstructured object properly decodes +// JSON when passed to Go's standard JSON library. +func (u *Unstructured) UnmarshalJSON(b []byte) error { + return json.Unmarshal(b, &u.Object) +} + +func (u *Unstructured) DeepCopy() *Unstructured { + if u == nil { + return nil + } + out := new(Unstructured) + u.DeepCopyInto(out) + return out +} + +func (u *Unstructured) DeepCopyInto(out *Unstructured) { + obj := map[string]any{} + if u.Object != nil { + jj, err := json.Marshal(u.Object) + if err == nil { + _ = json.Unmarshal(jj, &obj) + } + } + out.Object = obj +} diff --git a/experimental/apis/data/v0alpha1/zz_generated.deepcopy.go b/experimental/apis/data/v0alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..aaf8295c5 --- /dev/null +++ b/experimental/apis/data/v0alpha1/zz_generated.deepcopy.go @@ -0,0 +1,187 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v0alpha1 + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommonQueryProperties) DeepCopyInto(out *CommonQueryProperties) { + *out = *in + if in.ResultAssertions != nil { + in, out := &in.ResultAssertions, &out.ResultAssertions + *out = new(ResultAssertions) + (*in).DeepCopyInto(*out) + } + if in.TimeRange != nil { + in, out := &in.TimeRange, &out.TimeRange + *out = new(TimeRange) + **out = **in + } + if in.Datasource != nil { + in, out := &in.Datasource, &out.Datasource + *out = new(DataSourceRef) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommonQueryProperties. +func (in *CommonQueryProperties) DeepCopy() *CommonQueryProperties { + if in == nil { + return nil + } + out := new(CommonQueryProperties) + in.DeepCopyInto(out) + return out +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataQuery. +func (in *DataQuery) DeepCopy() *DataQuery { + if in == nil { + return nil + } + out := new(DataQuery) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueryDataRequest) DeepCopyInto(out *QueryDataRequest) { + *out = *in + out.TimeRange = in.TimeRange + if in.Queries != nil { + in, out := &in.Queries, &out.Queries + *out = make([]DataQuery, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryDataRequest. +func (in *QueryDataRequest) DeepCopy() *QueryDataRequest { + if in == nil { + return nil + } + out := new(QueryDataRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataSourceRef) DeepCopyInto(out *DataSourceRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataSourceRef. +func (in *DataSourceRef) DeepCopy() *DataSourceRef { + if in == nil { + return nil + } + out := new(DataSourceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DiscriminatorFieldValue) DeepCopyInto(out *DiscriminatorFieldValue) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiscriminatorFieldValue. +func (in *DiscriminatorFieldValue) DeepCopy() *DiscriminatorFieldValue { + if in == nil { + return nil + } + out := new(DiscriminatorFieldValue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueryExample) DeepCopyInto(out *QueryExample) { + *out = *in + in.SaveModel.DeepCopyInto(&out.SaveModel) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryExample. +func (in *QueryExample) DeepCopy() *QueryExample { + if in == nil { + return nil + } + out := new(QueryExample) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueryTypeDefinitionSpec) DeepCopyInto(out *QueryTypeDefinitionSpec) { + *out = *in + if in.Discriminators != nil { + in, out := &in.Discriminators, &out.Discriminators + *out = make([]DiscriminatorFieldValue, len(*in)) + copy(*out, *in) + } + in.Schema.DeepCopyInto(&out.Schema) + if in.Examples != nil { + in, out := &in.Examples, &out.Examples + *out = make([]QueryExample, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Changelog != nil { + in, out := &in.Changelog, &out.Changelog + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueryTypeDefinitionSpec. +func (in *QueryTypeDefinitionSpec) DeepCopy() *QueryTypeDefinitionSpec { + if in == nil { + return nil + } + out := new(QueryTypeDefinitionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResultAssertions) DeepCopyInto(out *ResultAssertions) { + *out = *in + out.TypeVersion = in.TypeVersion + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResultAssertions. +func (in *ResultAssertions) DeepCopy() *ResultAssertions { + if in == nil { + return nil + } + out := new(ResultAssertions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TimeRange) DeepCopyInto(out *TimeRange) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeRange. +func (in *TimeRange) DeepCopy() *TimeRange { + if in == nil { + return nil + } + out := new(TimeRange) + in.DeepCopyInto(out) + return out +}