Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ jobs:
timeout 300 go test -coverprofile=main-coverage.out -covermode=atomic ./... > /dev/null 2>&1 || echo "Main branch tests failed or timed out"

if [ -f main-coverage.out ]; then
# Filter out cmd and tests folders from main branch coverage (same as current branch)
grep -v -E '/cmd/|/tests/' main-coverage.out > main-coverage.filtered.out || true
mv main-coverage.filtered.out main-coverage.out

MAIN_COVERAGE=$(go tool cover -func=main-coverage.out | grep total | awk '{print $3}' || echo "0.0%")
echo "main-coverage=$MAIN_COVERAGE" >> $GITHUB_OUTPUT
echo "Main branch coverage: $MAIN_COVERAGE"
Expand Down
23 changes: 19 additions & 4 deletions marshaller/coremodel.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,16 @@ type CoreModeler interface {
SetConfig(config *yml.Config)
GetConfig() *yml.Config
Marshal(ctx context.Context, w io.Writer) error
SetUnknownProperties(props []string)
GetUnknownProperties() []string
}

type CoreModel struct {
RootNode *yaml.Node // RootNode is the node that was unmarshaled into this model
Valid bool // Valid indicates whether the model passed validation, ie all its required fields were present and ValidYaml is true
ValidYaml bool // ValidYaml indicates whether the model's underlying YAML representation is valid, for example a mapping node was received for a model
Config *yml.Config // Generally only set on the top-level model that was unmarshaled
RootNode *yaml.Node // RootNode is the node that was unmarshaled into this model
Valid bool // Valid indicates whether the model passed validation, ie all its required fields were present and ValidYaml is true
ValidYaml bool // ValidYaml indicates whether the model's underlying YAML representation is valid, for example a mapping node was received for a model
Config *yml.Config // Generally only set on the top-level model that was unmarshaled
UnknownProperties []string // UnknownProperties lists property keys that were present in the YAML but not defined in the model (excludes extensions which start with "x-")
}

var _ CoreModeler = (*CoreModel)(nil)
Expand Down Expand Up @@ -86,6 +89,18 @@ func (c *CoreModel) GetConfig() *yml.Config {
return c.Config
}

func (c *CoreModel) SetUnknownProperties(props []string) {
c.UnknownProperties = props
}

func (c *CoreModel) GetUnknownProperties() []string {
if c.UnknownProperties == nil {
return []string{}
}

return c.UnknownProperties
}

// GetJSONPointer returns the JSON pointer path from the topLevelRootNode to this CoreModel's RootNode.
// Returns an empty string if the node is not found or if either node is nil.
// The returned pointer follows RFC6901 format (e.g., "/path/to/node").
Expand Down
19 changes: 17 additions & 2 deletions marshaller/unmarshaller.go
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,10 @@ func unmarshalModel(ctx context.Context, parentName string, node *yaml.Node, str

jobValidationErrs := make([][]error, numJobs)

// Track unknown properties (non-extension, non-field, non-embedded map properties)
var unknownPropertiesMutex sync.Mutex
unknownProperties := make([]string, 0, numJobs)

// Mutex to protect concurrent access to extensionsField
var extensionsMutex sync.Mutex

Expand All @@ -363,15 +367,16 @@ func unmarshalModel(ctx context.Context, parentName string, node *yaml.Node, str
// Direct field index lookup (eliminates map[string]Field allocation)
fieldIndex, ok := fieldMap.FieldIndexes[key]
if !ok {
if strings.HasPrefix(key, "x-") && extensionsField != nil {
switch {
case strings.HasPrefix(key, "x-") && extensionsField != nil:
// Lock access to extensionsField to prevent concurrent modification
extensionsMutex.Lock()
defer extensionsMutex.Unlock()
err := UnmarshalExtension(keyNode, valueNode, *extensionsField)
if err != nil {
return err
}
} else if embeddedMap != nil {
case embeddedMap != nil:
// Skip alias definitions - these are nodes where:
// 1. The value node has an anchor (e.g., &keyAlias)
// 2. The key is not an alias reference (doesn't start with *)
Expand All @@ -381,6 +386,11 @@ func unmarshalModel(ctx context.Context, parentName string, node *yaml.Node, str
return nil
}
jobMapContent[i/2] = append(jobMapContent[i/2], keyNode, valueNode)
default:
// This is an unknown property (not a recognized field, not an extension, not in embedded map)
unknownPropertiesMutex.Lock()
unknownProperties = append(unknownProperties, key)
unknownPropertiesMutex.Unlock()
}
} else {
// Get field info from cache and field value directly
Expand Down Expand Up @@ -438,6 +448,11 @@ func unmarshalModel(ctx context.Context, parentName string, node *yaml.Node, str
validationErrs = append(validationErrs, embeddedMapValidationErrs...)
}

// Store unknown properties in the core model if any were found
if len(unknownProperties) > 0 {
unmarshallable.SetUnknownProperties(unknownProperties)
}

// Use the errors to determine the validity of the model
unmarshallable.DetermineValidity(validationErrs)

Expand Down
22 changes: 21 additions & 1 deletion mise-tasks/test-cli
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ $CLI spec validate --help > /dev/null
$CLI spec upgrade --help > /dev/null
$CLI spec inline --help > /dev/null
$CLI spec clean --help > /dev/null
$CLI spec sanitize --help > /dev/null
$CLI spec bundle --help > /dev/null
$CLI spec join --help > /dev/null
$CLI spec bootstrap --help > /dev/null
Expand Down Expand Up @@ -122,6 +123,23 @@ if ! diff -q dist/test/test-cleaned-empty.yaml openapi/testdata/clean/clean_empt
exit 1
fi

# Test sanitize command with known test files
echo " ✓ Testing sanitize command..."
$CLI spec sanitize openapi/testdata/sanitize/sanitize_input.yaml dist/test/test-sanitized.yaml > /dev/null
$CLI spec sanitize --config openapi/testdata/sanitize/sanitize_pattern_config.yaml openapi/testdata/sanitize/sanitize_pattern_input.yaml dist/test/test-sanitized-pattern.yaml > /dev/null

# Compare sanitize outputs with expected
echo " ✓ Comparing sanitize outputs with expected..."
if ! diff -q dist/test/test-sanitized.yaml openapi/testdata/sanitize/sanitize_expected.yaml > /dev/null; then
echo " ❌ Sanitize output differs from expected"
exit 1
fi

if ! diff -q dist/test/test-sanitized-pattern.yaml openapi/testdata/sanitize/sanitize_pattern_expected.yaml > /dev/null; then
echo " ❌ Sanitize pattern output differs from expected"
exit 1
fi

# Test join command with known test files
echo " ✓ Testing join command..."
$CLI spec join openapi/testdata/join/main.yaml openapi/testdata/join/subdir/second.yaml openapi/testdata/join/third.yaml dist/test/test-joined-counter.yaml > /dev/null
Expand Down Expand Up @@ -161,6 +179,8 @@ $CLI spec validate dist/test/test-joined-counter.yaml > /dev/null
$CLI spec validate dist/test/test-joined-filepath.yaml > /dev/null
$CLI spec validate dist/test/test-cleaned.yaml > /dev/null
$CLI spec validate dist/test/test-cleaned-empty.yaml > /dev/null
$CLI spec validate dist/test/test-sanitized.yaml > /dev/null
$CLI spec validate dist/test/test-sanitized-pattern.yaml > /dev/null
$CLI spec validate dist/test/test-joined-conflicts.yaml > /dev/null

# Test arazzo validation with known test files
Expand Down Expand Up @@ -232,7 +252,7 @@ echo "✅ All CLI integration tests passed!"
echo "📊 Test summary:"
echo " - Tested all command help outputs"
echo " - Validated known good and bad files"
echo " - Tested bootstrap, upgrade, inline, clean, bundle, join commands"
echo " - Tested bootstrap, upgrade, inline, clean, sanitize, bundle, join commands"
echo " - Compared outputs with expected results"
echo " - Tested arazzo validation"
echo " - Tested overlay validation, apply, and compare"
Expand Down
8 changes: 8 additions & 0 deletions mise-tasks/test-coverage
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ if ! gotestsum --format testname -- -race -coverprofile=coverage.out -covermode=
exit 1
fi

# Filter out cmd and tests folders from coverage report
if [ -f coverage.out ]; then
echo "🔧 Filtering cmd and tests folders from coverage report..."
grep -v -E '/cmd/|/tests/' coverage.out > coverage.filtered.out || true
# Keep original for reference, use filtered for reporting
mv coverage.filtered.out coverage.out
fi

echo ""
echo "## 📊 Test Coverage Report"
echo ""
Expand Down
161 changes: 161 additions & 0 deletions openapi/cmd/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ OpenAPI specifications define REST APIs in a standard format. These commands hel
- [`upgrade`](#upgrade)
- [`inline`](#inline)
- [`clean`](#clean)
- [`sanitize`](#sanitize)
- [`bundle`](#bundle)
- [Bundle vs Inline](#bundle-vs-inline)
- [`join`](#join)
Expand Down Expand Up @@ -216,6 +217,166 @@ components:
- You're preparing a specification for publication or distribution
- You want to reduce document size and complexity
- You're maintaining a large specification with many components

### `sanitize`

Remove unwanted elements from an OpenAPI specification to create clean, standards-compliant documents.

```bash
# Default sanitization (remove all extensions and unused components)
openapi spec sanitize ./spec.yaml

# Sanitize and write to new file
openapi spec sanitize ./spec.yaml ./clean-spec.yaml

# Sanitize in-place
openapi spec sanitize -w ./spec.yaml

# Use config file for selective sanitization
openapi spec sanitize --config sanitize-config.yaml ./spec.yaml
```

**Default Behavior (no config):**

By default, sanitize performs aggressive cleanup:

- Removes ALL x-* vendor extensions throughout the document
- Removes unused components (schemas, responses, parameters, etc.)
- Removes unknown properties not defined in the OpenAPI specification

**Configuration File Support:**

Create a YAML configuration file to control sanitization behavior:

```yaml
# sanitize-config.yaml

# Only remove extensions that match these patterns, null will remove ALL extensions, [] will remove no extensions (default: null, removes ALL extensions)
extensionPatterns:
- "x-go-*"
- "x-internal-*"

# Keep unused components (default: false, removes them)
keepUnusedComponents: true

# Keep unknown properties (default: false, removes them)
keepUnknownProperties: true
```

**What gets sanitized:**

- **Extensions**: All x-* vendor extensions (info, paths, operations, schemas, etc.)
- **Unused Components**: Schemas, responses, parameters, examples, request bodies, headers, security schemes, links, callbacks, and path items that aren't referenced
- **Unknown Properties**: Properties not defined in the OpenAPI specification

**Before sanitization:**

```yaml
openapi: 3.1.0
info:
title: My API
version: 1.0.0
x-api-id: internal-123
x-go-package: myapi
paths:
/users:
get:
operationId: listUsers
x-go-name: ListUsers
x-rate-limit: 100
responses:
'200':
description: Success
content:
application/json:
schema:
$ref: '#/components/schemas/User'
components:
schemas:
User:
type: object
x-go-type: User
properties:
id:
type: string
UnusedSchema:
type: object
description: Not referenced anywhere
```

**After sanitization (default):**

```yaml
openapi: 3.1.0
info:
title: My API
version: 1.0.0
paths:
/users:
get:
operationId: listUsers
responses:
'200':
description: Success
content:
application/json:
schema:
$ref: '#/components/schemas/User'
components:
schemas:
User:
type: object
properties:
id:
type: string
```

**After sanitization (with pattern config):**

Using config with `extensionPatterns: ["x-go-*"]`:

```yaml
openapi: 3.1.0
info:
title: My API
version: 1.0.0
x-api-id: internal-123 # kept (doesn't match x-go-*)
paths:
/users:
get:
operationId: listUsers
x-rate-limit: 100 # kept (doesn't match x-go-*)
responses:
'200':
description: Success
content:
application/json:
schema:
$ref: '#/components/schemas/User'
components:
schemas:
User:
type: object
properties:
id:
type: string
```

**Benefits of sanitization:**

- **Standards compliance**: Remove vendor-specific extensions for clean, standard specs
- **Clean distribution**: Prepare specifications for public sharing or publishing
- **Reduced size**: Remove unnecessary extensions and unused components
- **Selective cleanup**: Use patterns to target specific extension families
- **Flexible control**: Config file allows fine-grained control over what to keep

**Use Sanitize when:**

- You want to remove all vendor extensions before publishing
- You're preparing specifications for standards-compliant distribution
- You need to clean up internal annotations before sharing externally
- You want to remove specific extension families (e.g., x-go-*, x-internal-*)
- You're combining extension removal with component cleanup in one operation

### `bundle`

Expand Down
1 change: 1 addition & 0 deletions openapi/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ func Apply(rootCmd *cobra.Command) {
rootCmd.AddCommand(upgradeCmd)
rootCmd.AddCommand(inlineCmd)
rootCmd.AddCommand(cleanCmd)
rootCmd.AddCommand(sanitizeCmd)
rootCmd.AddCommand(bundleCmd)
rootCmd.AddCommand(joinCmd)
rootCmd.AddCommand(bootstrapCmd)
Expand Down
Loading