Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ jobs:
timeout 300 go test -coverprofile=main-coverage.out -covermode=atomic ./... > /dev/null 2>&1 || echo "Main branch tests failed or timed out"

if [ -f main-coverage.out ]; then
# Filter out cmd and tests folders from main branch coverage (same as current branch)
grep -v -E '/cmd/|/tests/' main-coverage.out > main-coverage.filtered.out || true
mv main-coverage.filtered.out main-coverage.out

MAIN_COVERAGE=$(go tool cover -func=main-coverage.out | grep total | awk '{print $3}' || echo "0.0%")
echo "main-coverage=$MAIN_COVERAGE" >> $GITHUB_OUTPUT
echo "Main branch coverage: $MAIN_COVERAGE"
Expand Down
23 changes: 19 additions & 4 deletions marshaller/coremodel.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,16 @@ type CoreModeler interface {
SetConfig(config *yml.Config)
GetConfig() *yml.Config
Marshal(ctx context.Context, w io.Writer) error
SetUnknownProperties(props []string)
GetUnknownProperties() []string
}

type CoreModel struct {
RootNode *yaml.Node // RootNode is the node that was unmarshaled into this model
Valid bool // Valid indicates whether the model passed validation, ie all its required fields were present and ValidYaml is true
ValidYaml bool // ValidYaml indicates whether the model's underlying YAML representation is valid, for example a mapping node was received for a model
Config *yml.Config // Generally only set on the top-level model that was unmarshaled
RootNode *yaml.Node // RootNode is the node that was unmarshaled into this model
Valid bool // Valid indicates whether the model passed validation, ie all its required fields were present and ValidYaml is true
ValidYaml bool // ValidYaml indicates whether the model's underlying YAML representation is valid, for example a mapping node was received for a model
Config *yml.Config // Generally only set on the top-level model that was unmarshaled
UnknownProperties []string // UnknownProperties lists property keys that were present in the YAML but not defined in the model (excludes extensions which start with "x-")
}

var _ CoreModeler = (*CoreModel)(nil)
Expand Down Expand Up @@ -86,6 +89,18 @@ func (c *CoreModel) GetConfig() *yml.Config {
return c.Config
}

func (c *CoreModel) SetUnknownProperties(props []string) {
c.UnknownProperties = props
}

func (c *CoreModel) GetUnknownProperties() []string {
if c.UnknownProperties == nil {
return []string{}
}

return c.UnknownProperties
}

// GetJSONPointer returns the JSON pointer path from the topLevelRootNode to this CoreModel's RootNode.
// Returns an empty string if the node is not found or if either node is nil.
// The returned pointer follows RFC6901 format (e.g., "/path/to/node").
Expand Down
19 changes: 17 additions & 2 deletions marshaller/unmarshaller.go
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,10 @@ func unmarshalModel(ctx context.Context, parentName string, node *yaml.Node, str

jobValidationErrs := make([][]error, numJobs)

// Track unknown properties (non-extension, non-field, non-embedded map properties)
var unknownPropertiesMutex sync.Mutex
var unknownProperties []string

// Mutex to protect concurrent access to extensionsField
var extensionsMutex sync.Mutex

Expand All @@ -363,15 +367,16 @@ func unmarshalModel(ctx context.Context, parentName string, node *yaml.Node, str
// Direct field index lookup (eliminates map[string]Field allocation)
fieldIndex, ok := fieldMap.FieldIndexes[key]
if !ok {
if strings.HasPrefix(key, "x-") && extensionsField != nil {
switch {
case strings.HasPrefix(key, "x-") && extensionsField != nil:
// Lock access to extensionsField to prevent concurrent modification
extensionsMutex.Lock()
defer extensionsMutex.Unlock()
err := UnmarshalExtension(keyNode, valueNode, *extensionsField)
if err != nil {
return err
}
} else if embeddedMap != nil {
case embeddedMap != nil:
// Skip alias definitions - these are nodes where:
// 1. The value node has an anchor (e.g., &keyAlias)
// 2. The key is not an alias reference (doesn't start with *)
Expand All @@ -381,6 +386,11 @@ func unmarshalModel(ctx context.Context, parentName string, node *yaml.Node, str
return nil
}
jobMapContent[i/2] = append(jobMapContent[i/2], keyNode, valueNode)
default:
// This is an unknown property (not a recognized field, not an extension, not in embedded map)
unknownPropertiesMutex.Lock()
unknownProperties = append(unknownProperties, key)
unknownPropertiesMutex.Unlock()
}
} else {
// Get field info from cache and field value directly
Expand Down Expand Up @@ -438,6 +448,11 @@ func unmarshalModel(ctx context.Context, parentName string, node *yaml.Node, str
validationErrs = append(validationErrs, embeddedMapValidationErrs...)
}

// Store unknown properties in the core model if any were found
if len(unknownProperties) > 0 {
unmarshallable.SetUnknownProperties(unknownProperties)
}

// Use the errors to determine the validity of the model
unmarshallable.DetermineValidity(validationErrs)

Expand Down
22 changes: 21 additions & 1 deletion mise-tasks/test-cli
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ $CLI spec validate --help > /dev/null
$CLI spec upgrade --help > /dev/null
$CLI spec inline --help > /dev/null
$CLI spec clean --help > /dev/null
$CLI spec sanitize --help > /dev/null
$CLI spec bundle --help > /dev/null
$CLI spec join --help > /dev/null
$CLI spec bootstrap --help > /dev/null
Expand Down Expand Up @@ -122,6 +123,23 @@ if ! diff -q dist/test/test-cleaned-empty.yaml openapi/testdata/clean/clean_empt
exit 1
fi

# Test sanitize command with known test files
echo " ✓ Testing sanitize command..."
$CLI spec sanitize openapi/testdata/sanitize/sanitize_input.yaml dist/test/test-sanitized.yaml > /dev/null
$CLI spec sanitize --config openapi/testdata/sanitize/sanitize_pattern_config.yaml openapi/testdata/sanitize/sanitize_pattern_input.yaml dist/test/test-sanitized-pattern.yaml > /dev/null

# Compare sanitize outputs with expected
echo " ✓ Comparing sanitize outputs with expected..."
if ! diff -q dist/test/test-sanitized.yaml openapi/testdata/sanitize/sanitize_expected.yaml > /dev/null; then
echo " ❌ Sanitize output differs from expected"
exit 1
fi

if ! diff -q dist/test/test-sanitized-pattern.yaml openapi/testdata/sanitize/sanitize_pattern_expected.yaml > /dev/null; then
echo " ❌ Sanitize pattern output differs from expected"
exit 1
fi

# Test join command with known test files
echo " ✓ Testing join command..."
$CLI spec join openapi/testdata/join/main.yaml openapi/testdata/join/subdir/second.yaml openapi/testdata/join/third.yaml dist/test/test-joined-counter.yaml > /dev/null
Expand Down Expand Up @@ -161,6 +179,8 @@ $CLI spec validate dist/test/test-joined-counter.yaml > /dev/null
$CLI spec validate dist/test/test-joined-filepath.yaml > /dev/null
$CLI spec validate dist/test/test-cleaned.yaml > /dev/null
$CLI spec validate dist/test/test-cleaned-empty.yaml > /dev/null
$CLI spec validate dist/test/test-sanitized.yaml > /dev/null
$CLI spec validate dist/test/test-sanitized-pattern.yaml > /dev/null
$CLI spec validate dist/test/test-joined-conflicts.yaml > /dev/null

# Test arazzo validation with known test files
Expand Down Expand Up @@ -232,7 +252,7 @@ echo "✅ All CLI integration tests passed!"
echo "📊 Test summary:"
echo " - Tested all command help outputs"
echo " - Validated known good and bad files"
echo " - Tested bootstrap, upgrade, inline, clean, bundle, join commands"
echo " - Tested bootstrap, upgrade, inline, clean, sanitize, bundle, join commands"
echo " - Compared outputs with expected results"
echo " - Tested arazzo validation"
echo " - Tested overlay validation, apply, and compare"
Expand Down
8 changes: 8 additions & 0 deletions mise-tasks/test-coverage
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ if ! gotestsum --format testname -- -race -coverprofile=coverage.out -covermode=
exit 1
fi

# Filter out cmd and tests folders from coverage report
if [ -f coverage.out ]; then
echo "🔧 Filtering cmd and tests folders from coverage report..."
grep -v -E '/cmd/|/tests/' coverage.out > coverage.filtered.out || true
# Keep original for reference, use filtered for reporting
mv coverage.filtered.out coverage.out
fi

echo ""
echo "## 📊 Test Coverage Report"
echo ""
Expand Down
161 changes: 161 additions & 0 deletions openapi/cmd/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ OpenAPI specifications define REST APIs in a standard format. These commands hel
- [`upgrade`](#upgrade)
- [`inline`](#inline)
- [`clean`](#clean)
- [`sanitize`](#sanitize)
- [`bundle`](#bundle)
- [Bundle vs Inline](#bundle-vs-inline)
- [`join`](#join)
Expand Down Expand Up @@ -216,6 +217,166 @@ components:
- You're preparing a specification for publication or distribution
- You want to reduce document size and complexity
- You're maintaining a large specification with many components

### `sanitize`

Remove unwanted elements from an OpenAPI specification to create clean, standards-compliant documents.

```bash
# Default sanitization (remove all extensions and unused components)
openapi spec sanitize ./spec.yaml

# Sanitize and write to new file
openapi spec sanitize ./spec.yaml ./clean-spec.yaml

# Sanitize in-place
openapi spec sanitize -w ./spec.yaml

# Use config file for selective sanitization
openapi spec sanitize --config sanitize-config.yaml ./spec.yaml
```

**Default Behavior (no config):**

By default, sanitize performs aggressive cleanup:

- Removes ALL x-* vendor extensions throughout the document
- Removes unused components (schemas, responses, parameters, etc.)
- Removes unknown properties not defined in the OpenAPI specification

**Configuration File Support:**

Create a YAML configuration file to control sanitization behavior:

```yaml
# sanitize-config.yaml

# Remove only specific extension patterns (if not set, removes ALL extensions)
extensionPatterns:
- "x-go-*"
- "x-internal-*"

# Keep unused components (default: false, removes them)
keepUnusedComponents: true

# Keep unknown properties (default: false, removes them)
keepUnknownProperties: true
```

**What gets sanitized:**

- **Extensions**: All x-* vendor extensions (info, paths, operations, schemas, etc.)
- **Unused Components**: Schemas, responses, parameters, examples, request bodies, headers, security schemes, links, callbacks, and path items that aren't referenced
- **Unknown Properties**: Properties not defined in the OpenAPI specification

**Before sanitization:**

```yaml
openapi: 3.1.0
info:
title: My API
version: 1.0.0
x-api-id: internal-123
x-go-package: myapi
paths:
/users:
get:
operationId: listUsers
x-go-name: ListUsers
x-rate-limit: 100
responses:
'200':
description: Success
content:
application/json:
schema:
$ref: '#/components/schemas/User'
components:
schemas:
User:
type: object
x-go-type: User
properties:
id:
type: string
UnusedSchema:
type: object
description: Not referenced anywhere
```

**After sanitization (default):**

```yaml
openapi: 3.1.0
info:
title: My API
version: 1.0.0
paths:
/users:
get:
operationId: listUsers
responses:
'200':
description: Success
content:
application/json:
schema:
$ref: '#/components/schemas/User'
components:
schemas:
User:
type: object
properties:
id:
type: string
```

**After sanitization (with pattern config):**

Using config with `extensionPatterns: ["x-go-*"]`:

```yaml
openapi: 3.1.0
info:
title: My API
version: 1.0.0
x-api-id: internal-123 # kept (doesn't match x-go-*)
paths:
/users:
get:
operationId: listUsers
x-rate-limit: 100 # kept (doesn't match x-go-*)
responses:
'200':
description: Success
content:
application/json:
schema:
$ref: '#/components/schemas/User'
components:
schemas:
User:
type: object
properties:
id:
type: string
```

**Benefits of sanitization:**

- **Standards compliance**: Remove vendor-specific extensions for clean, standard specs
- **Clean distribution**: Prepare specifications for public sharing or publishing
- **Reduced size**: Remove unnecessary extensions and unused components
- **Selective cleanup**: Use patterns to target specific extension families
- **Flexible control**: Config file allows fine-grained control over what to keep

**Use Sanitize when:**

- You want to remove all vendor extensions before publishing
- You're preparing specifications for standards-compliant distribution
- You need to clean up internal annotations before sharing externally
- You want to remove specific extension families (e.g., x-go-*, x-internal-*)
- You're combining extension removal with component cleanup in one operation

### `bundle`

Expand Down
1 change: 1 addition & 0 deletions openapi/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ func Apply(rootCmd *cobra.Command) {
rootCmd.AddCommand(upgradeCmd)
rootCmd.AddCommand(inlineCmd)
rootCmd.AddCommand(cleanCmd)
rootCmd.AddCommand(sanitizeCmd)
rootCmd.AddCommand(bundleCmd)
rootCmd.AddCommand(joinCmd)
rootCmd.AddCommand(bootstrapCmd)
Expand Down
Loading