From 5bfab16c4cb94e2363bddd6551f8e84724281d0e Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Thu, 13 Mar 2025 18:48:28 -0700 Subject: [PATCH 01/10] initial definition of extension manifest --- dsc_lib/locales/en-us.toml | 4 + dsc_lib/src/extensions/extension_manifest.rs | 154 +++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 dsc_lib/src/extensions/extension_manifest.rs diff --git a/dsc_lib/locales/en-us.toml b/dsc_lib/locales/en-us.toml index f0e03aab..ca012000 100644 --- a/dsc_lib/locales/en-us.toml +++ b/dsc_lib/locales/en-us.toml @@ -160,6 +160,10 @@ diffMissingItem = "diff: actual array missing expected item" resourceManifestSchemaTitle = "Resource manifest schema URI" resourceManifestSchemaDescription = "Defines the JSON Schema the resource manifest adheres to." +[extensions.extension_manifest] +extensionManifestSchemaTitle = "Extension manifest schema URI" +extensionManifestSchemaDescription = "Defines the JSON Schema the extension manifest adheres to." + [functions] invalidArgType = "Invalid argument type" invalidArguments = "Invalid argument(s)" diff --git a/dsc_lib/src/extensions/extension_manifest.rs b/dsc_lib/src/extensions/extension_manifest.rs new file mode 100644 index 00000000..44eabb3d --- /dev/null +++ b/dsc_lib/src/extensions/extension_manifest.rs @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::dscresources::resource_manifest::ArgKind; +use rust_i18n::t; +use schemars::JsonSchema; +use semver::Version; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; + +use crate::{dscerror::DscError, schemas::DscRepoSchema}; + +#[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ExtensionManifest { + /// The version of the resource manifest schema. + #[serde(rename = "$schema")] + #[schemars(schema_with = "ExtensionManifest::recognized_schema_uris_subschema")] + pub schema_version: String, + /// The namespaced name of the extension. + #[serde(rename = "type")] + pub r#type: String, + /// The version of the resource using semantic versioning. + pub version: String, + /// The description of the resource. + pub description: Option, + /// Tags for the resource. + pub tags: Option>, + /// Details how to call the Discover method of the resource. + pub discover: Option, + /// Mapping of exit codes to descriptions. Zero is always success and non-zero is always failure. + #[serde(rename = "exitCodes", skip_serializing_if = "Option::is_none")] + pub exit_codes: Option>, +} + +#[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize, JsonSchema)] +pub struct DiscoverMethod { + /// The command to run to get the state of the resource. + pub executable: String, + /// The arguments to pass to the command to perform a Get. + pub args: Option>, +} + +impl DscRepoSchema for ExtensionManifest { + const SCHEMA_FILE_BASE_NAME: &'static str = "manifest"; + const SCHEMA_FOLDER_PATH: &'static str = "extension"; + const SCHEMA_SHOULD_BUNDLE: bool = true; + + fn schema_metadata() -> schemars::schema::Metadata { + schemars::schema::Metadata { + title: Some(t!("extensions.extension_manifest.extensionManifestSchemaTitle").into()), + description: Some(t!("extensions.extension_manifest.extensioneManifestSchemaDescription").into()), + ..Default::default() + } + } + + fn validate_schema_uri(&self) -> Result<(), DscError> { + if Self::is_recognized_schema_uri(&self.schema_version) { + Ok(()) + } else { + Err(DscError::UnrecognizedSchemaUri( + self.schema_version.clone(), + Self::recognized_schema_uris(), + )) + } + } +} + +/// Import a resource manifest from a JSON value. +/// +/// # Arguments +/// +/// * `manifest` - The JSON value to import. +/// +/// # Returns +/// +/// * `Result` - The imported resource manifest. +/// +/// # Errors +/// +/// * `DscError` - The JSON value is invalid or the schema version is not supported. +pub fn import_manifest(manifest: Value) -> Result { + // TODO: enable schema version validation, if not provided, use the latest + let manifest = serde_json::from_value::(manifest)?; + Ok(manifest) +} + +/// Validate a semantic version string. +/// +/// # Arguments +/// +/// * `version` - The semantic version string to validate. +/// +/// # Returns +/// +/// * `Result<(), Error>` - The result of the validation. +/// +/// # Errors +/// +/// * `Error` - The version string is not a valid semantic version. +pub fn validate_semver(version: &str) -> Result<(), semver::Error> { + Version::parse(version)?; + Ok(()) +} + +#[cfg(test)] +mod test { + use crate::{ + dscerror::DscError, + dscresources::resource_manifest::ResourceManifest, + schemas::DscRepoSchema + }; + + #[test] + fn test_validate_schema_uri_with_invalid_uri() { + let invalid_uri = "https://invalid.schema.uri".to_string(); + + let manifest = ResourceManifest{ + schema_version: invalid_uri.clone(), + resource_type: "Microsoft.Dsc.Test/InvalidSchemaUri".to_string(), + version: "0.1.0".to_string(), + ..Default::default() + }; + + let ref result = manifest.validate_schema_uri(); + + assert!(result.as_ref().is_err()); + + match result.as_ref().unwrap_err() { + DscError::UnrecognizedSchemaUri(actual, recognized) => { + assert_eq!(actual, &invalid_uri); + assert_eq!(recognized, &ResourceManifest::recognized_schema_uris()) + }, + _ => { + panic!("Expected validate_schema_uri() to error on unrecognized schema uri, but was {:?}", result.as_ref().unwrap_err()) + } + } + } + + #[test] + fn test_validate_schema_uri_with_valid_uri() { + let manifest = ResourceManifest{ + schema_version: ResourceManifest::default_schema_id_uri(), + resource_type: "Microsoft.Dsc.Test/ValidSchemaUri".to_string(), + version: "0.1.0".to_string(), + ..Default::default() + }; + + let result = manifest.validate_schema_uri(); + + assert!(result.is_ok()); + } +} From 022e38de750e3d4c79bbfc66dab7d9cc6dd2da19 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Tue, 18 Mar 2025 12:33:52 -0700 Subject: [PATCH 02/10] add enum for discovery types --- dsc_lib/src/discovery/discovery_trait.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dsc_lib/src/discovery/discovery_trait.rs b/dsc_lib/src/discovery/discovery_trait.rs index a6eb226f..1c652bf7 100644 --- a/dsc_lib/src/discovery/discovery_trait.rs +++ b/dsc_lib/src/discovery/discovery_trait.rs @@ -4,6 +4,11 @@ use crate::{dscresources::dscresource::DscResource, dscerror::DscError}; use std::collections::BTreeMap; +pub enum DiscoveryKind { + Resource, + Extension, +} + pub trait ResourceDiscovery { fn discover_resources(&mut self, filter: &str) -> Result<(), DscError>; fn discover_adapted_resources(&mut self, name_filter: &str, adapter_filter: &str) -> Result<(), DscError>; From fec58d2dcc35aaf648f5a4cf70e94b39006c70f3 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Tue, 22 Apr 2025 22:01:15 -0700 Subject: [PATCH 03/10] enable discovery of extensions --- dsc/locales/en-us.toml | 2 + dsc/src/args.rs | 16 ++ dsc/src/main.rs | 3 + dsc/src/subcommand.rs | 217 +++++++++++++------ dsc_lib/locales/en-us.toml | 5 + dsc_lib/src/discovery/command_discovery.rs | 181 ++++++++++++---- dsc_lib/src/discovery/discovery_trait.rs | 9 +- dsc_lib/src/discovery/mod.rs | 19 +- dsc_lib/src/extensions/discover.rs | 20 ++ dsc_lib/src/extensions/dscextension.rs | 57 +++++ dsc_lib/src/extensions/extension_manifest.rs | 10 +- dsc_lib/src/extensions/mod.rs | 6 + dsc_lib/src/lib.rs | 7 +- 13 files changed, 414 insertions(+), 138 deletions(-) create mode 100644 dsc_lib/src/extensions/discover.rs create mode 100644 dsc_lib/src/extensions/dscextension.rs create mode 100644 dsc_lib/src/extensions/mod.rs diff --git a/dsc/locales/en-us.toml b/dsc/locales/en-us.toml index ab72129f..c546c5c5 100644 --- a/dsc/locales/en-us.toml +++ b/dsc/locales/en-us.toml @@ -10,6 +10,7 @@ configAbout = "Apply a configuration document" parameters = "Parameters to pass to the configuration as JSON or YAML" parametersFile = "Parameters to pass to the configuration as a JSON or YAML file" systemRoot = "Specify the operating system root path if not targeting the current running OS" +extensionAbout = "Operations on DSC extensions" resourceAbout = "Invoke a specific DSC resource" schemaAbout = "Get the JSON schema for a DSC type" schemaType = "The type of DSC schema to get" @@ -24,6 +25,7 @@ validateAbout = "Validate the current configuration" exportAbout = "Export the current configuration" resolveAbout = "Resolve the current configuration" listAbout = "List or find resources" +listExtensionAbout = "List or find extensions" adapter = "Adapter filter to limit the resource search" description = "Description keyword to search for in the resource description" tags = "Tag to search for in the resource tags" diff --git a/dsc/src/args.rs b/dsc/src/args.rs index 410ba2f6..31a3627f 100644 --- a/dsc/src/args.rs +++ b/dsc/src/args.rs @@ -64,6 +64,11 @@ pub enum SubCommand { #[clap(long, hide = true)] as_include: bool, }, + #[clap(name = "extension", about = t!("args.extensionAbout").to_string())] + Extension { + #[clap(subcommand)] + subcommand: ExtensionSubCommand, + }, #[clap(name = "resource", about = t!("args.resourceAbout").to_string())] Resource { #[clap(subcommand)] @@ -144,6 +149,17 @@ pub enum ConfigSubCommand { } } +#[derive(Debug, PartialEq, Eq, Subcommand)] +pub enum ExtensionSubCommand { + #[clap(name = "list", about = t!("args.listExtensionAbout").to_string())] + List { + /// Optional filter to apply to the list of extensions + extension_name: Option, + #[clap(short = 'o', long, help = t!("args.outputFormat").to_string())] + output_format: Option, + }, +} + #[derive(Debug, PartialEq, Eq, Subcommand)] pub enum ResourceSubCommand { #[clap(name = "list", about = t!("args.listAbout").to_string())] diff --git a/dsc/src/main.rs b/dsc/src/main.rs index 526c3e62..68a0fce6 100644 --- a/dsc/src/main.rs +++ b/dsc/src/main.rs @@ -64,6 +64,9 @@ fn main() { subcommand::config(&subcommand, ¶meters, system_root.as_ref(), &as_group, &as_assert, &as_include, progress_format); } }, + SubCommand::Extension { subcommand } => { + subcommand::extension(&subcommand, progress_format); + }, SubCommand::Resource { subcommand } => { subcommand::resource(&subcommand, progress_format); }, diff --git a/dsc/src/subcommand.rs b/dsc/src/subcommand.rs index cc1df6c0..a1a182de 100644 --- a/dsc/src/subcommand.rs +++ b/dsc/src/subcommand.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::args::{ConfigSubCommand, DscType, OutputFormat, ResourceSubCommand}; +use crate::args::{ConfigSubCommand, DscType, ExtensionSubCommand, OutputFormat, ResourceSubCommand}; use crate::resolve::{get_contents, Include}; use crate::resource_command::{get_resource, self}; use crate::tablewriter::Table; @@ -16,6 +16,8 @@ use dsc_lib::{ config_result::ResourceGetResult, Configurator, }, + discovery::discovery_trait::DiscoveryKind, + discovery::command_discovery::ManifestResource, dscerror::DscError, DscManager, dscresources::invoke_result::{ @@ -25,6 +27,7 @@ use dsc_lib::{ }, dscresources::dscresource::{Capability, ImplementedAs, Invoke}, dscresources::resource_manifest::{import_manifest, ResourceManifest}, + extensions::dscextension::Capability as ExtensionCapability, progress::ProgressFormat, }; use rust_i18n::t; @@ -543,6 +546,22 @@ pub fn validate_config(config: &Configuration, progress_format: ProgressFormat) Ok(()) } +pub fn extension(subcommand: &ExtensionSubCommand, progress_format: ProgressFormat) { + let mut dsc = match DscManager::new() { + Ok(dsc) => dsc, + Err(err) => { + error!("Error: {err}"); + exit(EXIT_DSC_ERROR); + } + }; + + match subcommand { + ExtensionSubCommand::List{extension_name, output_format} => { + list_extensions(&mut dsc, extension_name.as_ref(), output_format.as_ref(), progress_format); + }, + } +} + #[allow(clippy::too_many_lines)] pub fn resource(subcommand: &ResourceSubCommand, progress_format: ProgressFormat) { let mut dsc = match DscManager::new() { @@ -592,6 +611,62 @@ pub fn resource(subcommand: &ResourceSubCommand, progress_format: ProgressFormat } } +fn list_extensions(dsc: &mut DscManager, extension_name: Option<&String>, format: Option<&OutputFormat>, progress_format: ProgressFormat) { + let mut write_table = false; + let mut table = Table::new(&[ + t!("subcommand.tableHeader_type").to_string().as_ref(), + t!("subcommand.tableHeader_version").to_string().as_ref(), + t!("subcommand.tableHeader_capabilities").to_string().as_ref(), + t!("subcommand.tableHeader_description").to_string().as_ref(), + ]); + if format.is_none() && io::stdout().is_terminal() { + // write as table if format is not specified and interactive + write_table = true; + } + let mut include_separator = false; + for manifest_resource in dsc.list_available(&DiscoveryKind::Extension, extension_name.unwrap_or(&String::from("*")), &String::new(), progress_format) { + if let ManifestResource::Extension(extension) = manifest_resource { + let mut capabilities = "-".to_string(); + let capability_types = [ + (ExtensionCapability::Discover, "d"), + ]; + + for (i, (capability, letter)) in capability_types.iter().enumerate() { + if extension.capabilities.contains(capability) { + capabilities.replace_range(i..=i, letter); + } + } + + if write_table { + table.add_row(vec![ + extension.type_name, + extension.version, + capabilities, + extension.description.unwrap_or_default() + ]); + } + else { + // convert to json + let json = match serde_json::to_string(&extension) { + Ok(json) => json, + Err(err) => { + error!("JSON: {err}"); + exit(EXIT_JSON_ERROR); + } + }; + write_object(&json, format, include_separator); + include_separator = true; + // insert newline separating instances if writing to console + if io::stdout().is_terminal() { println!(); } + } + } + } + + if write_table { + table.print(); + } +} + fn list_resources(dsc: &mut DscManager, resource_name: Option<&String>, adapter_name: Option<&String>, description: Option<&String>, tags: Option<&Vec>, format: Option<&OutputFormat>, progress_format: ProgressFormat) { let mut write_table = false; let mut table = Table::new(&[ @@ -607,86 +682,88 @@ fn list_resources(dsc: &mut DscManager, resource_name: Option<&String>, adapter_ write_table = true; } let mut include_separator = false; - for resource in dsc.list_available_resources(resource_name.unwrap_or(&String::from("*")), adapter_name.unwrap_or(&String::new()), progress_format) { - let mut capabilities = "--------".to_string(); - let capability_types = [ - (Capability::Get, "g"), - (Capability::Set, "s"), - (Capability::SetHandlesExist, "x"), - (Capability::WhatIf, "w"), - (Capability::Test, "t"), - (Capability::Delete, "d"), - (Capability::Export, "e"), - (Capability::Resolve, "r"), - ]; - - for (i, (capability, letter)) in capability_types.iter().enumerate() { - if resource.capabilities.contains(capability) { - capabilities.replace_range(i..=i, letter); + for manifest_resource in dsc.list_available(&DiscoveryKind::Resource, resource_name.unwrap_or(&String::from("*")), adapter_name.unwrap_or(&String::new()), progress_format) { + if let ManifestResource::Resource(resource) = manifest_resource { + let mut capabilities = "--------".to_string(); + let capability_types = [ + (Capability::Get, "g"), + (Capability::Set, "s"), + (Capability::SetHandlesExist, "x"), + (Capability::WhatIf, "w"), + (Capability::Test, "t"), + (Capability::Delete, "d"), + (Capability::Export, "e"), + (Capability::Resolve, "r"), + ]; + + for (i, (capability, letter)) in capability_types.iter().enumerate() { + if resource.capabilities.contains(capability) { + capabilities.replace_range(i..=i, letter); + } } - } - // if description, tags, or write_table is specified, pull resource manifest if it exists - if let Some(ref resource_manifest) = resource.manifest { - let manifest = match import_manifest(resource_manifest.clone()) { - Ok(resource_manifest) => resource_manifest, - Err(err) => { - error!("{} {}: {err}", t!("subcommand.invalidManifest"), resource.type_name); + // if description, tags, or write_table is specified, pull resource manifest if it exists + if let Some(ref resource_manifest) = resource.manifest { + let manifest = match import_manifest(resource_manifest.clone()) { + Ok(resource_manifest) => resource_manifest, + Err(err) => { + error!("{} {}: {err}", t!("subcommand.invalidManifest"), resource.type_name); + continue; + } + }; + + // if description is specified, skip if resource description does not contain it + if description.is_some() && + (manifest.description.is_none() | !manifest.description.unwrap_or_default().to_lowercase().contains(&description.unwrap_or(&String::new()).to_lowercase())) { continue; } - }; - - // if description is specified, skip if resource description does not contain it - if description.is_some() && - (manifest.description.is_none() | !manifest.description.unwrap_or_default().to_lowercase().contains(&description.unwrap_or(&String::new()).to_lowercase())) { - continue; - } - // if tags is specified, skip if resource tags do not contain the tags - if let Some(tags) = tags { - let Some(manifest_tags) = manifest.tags else { continue; }; + // if tags is specified, skip if resource tags do not contain the tags + if let Some(tags) = tags { + let Some(manifest_tags) = manifest.tags else { continue; }; - let mut found = false; - for tag_to_find in tags { - for tag in &manifest_tags { - if tag.to_lowercase() == tag_to_find.to_lowercase() { - found = true; - break; + let mut found = false; + for tag_to_find in tags { + for tag in &manifest_tags { + if tag.to_lowercase() == tag_to_find.to_lowercase() { + found = true; + break; + } } } + if !found { continue; } + } + } else { + // resource does not have a manifest but filtering on description or tags was requested - skip such resource + if description.is_some() || tags.is_some() { + continue; } - if !found { continue; } - } - } else { - // resource does not have a manifest but filtering on description or tags was requested - skip such resource - if description.is_some() || tags.is_some() { - continue; } - } - if write_table { - table.add_row(vec![ - resource.type_name, - format!("{:?}", resource.kind), - resource.version, - capabilities, - resource.require_adapter.unwrap_or_default(), - resource.description.unwrap_or_default() - ]); - } - else { - // convert to json - let json = match serde_json::to_string(&resource) { - Ok(json) => json, - Err(err) => { - error!("JSON: {err}"); - exit(EXIT_JSON_ERROR); - } - }; - write_object(&json, format, include_separator); - include_separator = true; - // insert newline separating instances if writing to console - if io::stdout().is_terminal() { println!(); } + if write_table { + table.add_row(vec![ + resource.type_name, + format!("{:?}", resource.kind), + resource.version, + capabilities, + resource.require_adapter.unwrap_or_default(), + resource.description.unwrap_or_default() + ]); + } + else { + // convert to json + let json = match serde_json::to_string(&resource) { + Ok(json) => json, + Err(err) => { + error!("JSON: {err}"); + exit(EXIT_JSON_ERROR); + } + }; + write_object(&json, format, include_separator); + include_separator = true; + // insert newline separating instances if writing to console + if io::stdout().is_terminal() { println!(); } + } } } diff --git a/dsc_lib/locales/en-us.toml b/dsc_lib/locales/en-us.toml index ca012000..1ff30d77 100644 --- a/dsc_lib/locales/en-us.toml +++ b/dsc_lib/locales/en-us.toml @@ -67,6 +67,9 @@ parameterNotObject = "Parameter '%{name}' is not an object" invokePropertyExpressions = "Invoke property expressions" invokeExpression = "Invoke property expression for %{name}: %{value}" +[discovery.mod] +extensionUnexpcted = "Extension '%{extension}' not expected" + [discovery.commandDiscovery] couldNotReadSetting = "Could not read 'resourcePath' setting" appendingEnvPath = "Appending PATH to resourcePath" @@ -79,9 +82,11 @@ discoverResources = "Discovering resources using filter: %{filter}" invalidAdapterFilter = "Could not build Regex filter for adapter name" progressSearching = "Searching for resources" foundResourceManifest = "Found resource manifest: %{path}" +extensionFound = "Extension '%{extension}' found" adapterFound = "Resource adapter '%{adapter}' found" resourceFound = "Resource '%{resource}' found" executableNotFound = "Executable '%{executable}' not found for operation '%{operation}' for resource '%{resource}'" +extensionInvalidVersion = "Extension '%{extension}' version '%{version}' is invalid" [dscresources.commandResource] invokeGet = "Invoking get for '%{resource}'" diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index ec556b1b..9e95a2a1 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::discovery::discovery_trait::ResourceDiscovery; +use crate::discovery::discovery_trait::{ResourceDiscovery, DiscoveryKind}; use crate::discovery::convert_wildcard_to_regex; use crate::dscresources::dscresource::{Capability, DscResource, ImplementedAs}; use crate::dscresources::resource_manifest::{import_manifest, validate_semver, Kind, ResourceManifest, SchemaKind}; use crate::dscresources::command_resource::invoke_command; use crate::dscerror::DscError; +use crate::extensions::dscextension::{self, DscExtension}; +use crate::extensions::extension_manifest::ExtensionManifest; use crate::progress::{ProgressBar, ProgressFormat}; use linked_hash_map::LinkedHashMap; use regex::RegexBuilder; @@ -17,8 +19,6 @@ use std::collections::{BTreeMap, HashSet, HashMap}; use std::env; use std::ffi::OsStr; use std::fs; -use std::fs::File; -use std::io::BufReader; use std::path::{Path, PathBuf}; use std::str::FromStr; use tracing::{debug, info, trace, warn}; @@ -27,10 +27,20 @@ use which::which; use crate::util::get_setting; use crate::util::get_exe_path; +const DSC_RESOURCE_EXTENSIONS: [&str; 3] = [".dsc.resource.json", ".dsc.resource.yaml", ".dsc.resource.yml"]; +const DSC_EXTENSION_EXTENSIONS: [&str; 3] = [".dsc.extension.json", ".dsc.extension.yaml", ".dsc.extension.yml"]; + +#[derive(Clone)] +pub enum ManifestResource { + Resource(DscResource), + Extension(DscExtension), +} + pub struct CommandDiscovery { // use BTreeMap so that the results are sorted by the typename, the Vec is sorted by version - resources: BTreeMap>, adapters: BTreeMap>, + resources: BTreeMap>, + extensions: BTreeMap, adapted_resources: BTreeMap>, progress_format: ProgressFormat, } @@ -60,8 +70,9 @@ impl Default for ResourcePathSetting { impl CommandDiscovery { pub fn new(progress_format: ProgressFormat) -> CommandDiscovery { CommandDiscovery { - resources: BTreeMap::new(), adapters: BTreeMap::new(), + resources: BTreeMap::new(), + extensions: BTreeMap::new(), adapted_resources: BTreeMap::new(), progress_format, } @@ -170,7 +181,7 @@ impl Default for CommandDiscovery { impl ResourceDiscovery for CommandDiscovery { - fn discover_resources(&mut self, filter: &str) -> Result<(), DscError> { + fn discover(&mut self, kind: &DiscoveryKind, filter: &str) -> Result<(), DscError> { info!("{}", t!("discovery.commandDiscovery.discoverResources", filter = filter)); let regex_str = convert_wildcard_to_regex(filter); @@ -184,8 +195,9 @@ impl ResourceDiscovery for CommandDiscovery { let mut progress = ProgressBar::new(1, self.progress_format)?; progress.write_activity(t!("discovery.commandDiscovery.progressSearching").to_string().as_str()); - let mut resources = BTreeMap::>::new(); let mut adapters = BTreeMap::>::new(); + let mut resources = BTreeMap::>::new(); + let mut extensions = BTreeMap::::new(); if let Ok(paths) = CommandDiscovery::get_resource_paths() { for path in paths { @@ -203,10 +215,9 @@ impl ResourceDiscovery for CommandDiscovery { continue; }; let file_name_lowercase = file_name.to_lowercase(); - if file_name_lowercase.ends_with(".dsc.resource.json") || - file_name_lowercase.ends_with(".dsc.resource.yaml") || - file_name_lowercase.ends_with(".dsc.resource.yml") { - trace!("{}", t!("discovery.commandDiscovery.foundResourceManifest", path = path.to_string_lossy())); + if (kind == &DiscoveryKind::Resource && DSC_RESOURCE_EXTENSIONS.iter().any(|ext| file_name_lowercase.ends_with(ext))) || + (kind == &DiscoveryKind::Extension && DSC_EXTENSION_EXTENSIONS.iter().any(|ext| file_name_lowercase.ends_with(ext))) { + trace!("{}", t!("discovery.commandDiscovery.foundManifest", path = path.to_string_lossy())); let resource = match load_manifest(&path) { Ok(r) => r, @@ -220,15 +231,38 @@ impl ResourceDiscovery for CommandDiscovery { }, }; - if regex.is_match(&resource.type_name) { - if let Some(ref manifest) = resource.manifest { - let manifest = import_manifest(manifest.clone())?; - if manifest.kind == Some(Kind::Adapter) { - trace!("{}", t!("discovery.commandDiscovery.adapterFound", adapter = resource.type_name)); - insert_resource(&mut adapters, &resource, true); - } else { - trace!("{}", t!("discovery.commandDiscovery.resourceFound", resource = resource.type_name)); - insert_resource(&mut resources, &resource, true); + match resource { + ManifestResource::Extension(extension) => { + if regex.is_match(&extension.type_name) { + trace!("{}", t!("discovery.commandDiscovery.extensionFound", extension = extension.type_name)); + // we only keep newest version of the extension so compare the version and only keep the newest + if let Some(existing_extension) = extensions.get_mut(&extension.type_name) { + let Ok(existing_version) = Version::parse(&existing_extension.version) else { + return Err(DscError::Operation(t!("discovery.commandDiscovery.extensionInvalidVersion", extension = existing_extension.type_name, version = existing_extension.version).to_string())); + }; + let Ok(new_version) = Version::parse(&extension.version) else { + return Err(DscError::Operation(t!("discovery.commandDiscovery.extensionInvalidVersion", extension = extension.type_name, version = extension.version).to_string())); + }; + if new_version > existing_version { + extensions.insert(extension.type_name.clone(), extension.clone()); + } + } else { + extensions.insert(extension.type_name.clone(), extension.clone()); + } + } + }, + ManifestResource::Resource(resource) => { + if regex.is_match(&resource.type_name) { + if let Some(ref manifest) = resource.manifest { + let manifest = import_manifest(manifest.clone())?; + if manifest.kind == Some(Kind::Adapter) { + trace!("{}", t!("discovery.commandDiscovery.adapterFound", adapter = resource.type_name)); + insert_resource(&mut adapters, &resource, true); + } else { + trace!("{}", t!("discovery.commandDiscovery.resourceFound", resource = resource.type_name)); + insert_resource(&mut resources, &resource, true); + } + } } } } @@ -238,16 +272,18 @@ impl ResourceDiscovery for CommandDiscovery { } } } + progress.write_increment(1); debug!("Found {} matching non-adapter-based resources", resources.len()); - self.resources = resources; self.adapters = adapters; + self.resources = resources; + self.extensions = extensions; Ok(()) } fn discover_adapted_resources(&mut self, name_filter: &str, adapter_filter: &str) -> Result<(), DscError> { if self.resources.is_empty() && self.adapters.is_empty() { - self.discover_resources("*")?; + self.discover(&DiscoveryKind::Resource, "*")?; } if self.adapters.is_empty() { @@ -352,24 +388,36 @@ impl ResourceDiscovery for CommandDiscovery { Ok(()) } - fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError> { + fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError> { trace!("Listing resources with type_name_filter '{type_name_filter}' and adapter_name_filter '{adapter_name_filter}'"); - let mut resources = BTreeMap::>::new(); + let mut resources = BTreeMap::>::new(); - if adapter_name_filter.is_empty() { - self.discover_resources(type_name_filter)?; - resources.append(&mut self.resources); - resources.append(&mut self.adapters); - } else { - self.discover_resources("*")?; - self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; + if *kind == DiscoveryKind::Resource { + if adapter_name_filter.is_empty() { + self.discover(kind, type_name_filter)?; + for (resource_name, resources_vec) in &self.resources { + resources.insert(resource_name.clone(), resources_vec.iter().map(|r| ManifestResource::Resource(r.clone())).collect()); + } + for (adapter_name, adapter_vec) in &self.adapters { + resources.insert(adapter_name.clone(), adapter_vec.iter().map(|r| ManifestResource::Resource(r.clone())).collect()); + } + } else { + self.discover(kind, "*")?; + self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; - // add/update found adapted resources to the lookup_table - add_resources_to_lookup_table(&self.adapted_resources); + // add/update found adapted resources to the lookup_table + add_resources_to_lookup_table(&self.adapted_resources); - // note: in next line 'BTreeMap::append' will leave self.adapted_resources empty - resources.append(&mut self.adapted_resources); + for (adapted_name, adapted_vec) in &self.adapted_resources { + resources.insert(adapted_name.clone(), adapted_vec.iter().map(|r| ManifestResource::Resource(r.clone())).collect()); + } + } + } else { + self.discover(kind, type_name_filter)?; + for (extension_name, extension) in &self.extensions { + resources.insert(extension_name.clone(), vec![ManifestResource::Extension(extension.clone())]); + } } Ok(resources) @@ -379,7 +427,7 @@ impl ResourceDiscovery for CommandDiscovery { fn find_resources(&mut self, required_resource_types: &[String]) -> Result, DscError> { debug!("Searching for resources: {:?}", required_resource_types); - self.discover_resources("*")?; + self.discover( &DiscoveryKind::Resource, "*")?; // convert required_resource_types to lowercase to handle case-insentiive search let mut remaining_required_resource_types = required_resource_types.iter().map(|x| x.to_lowercase()).collect::>(); @@ -458,6 +506,7 @@ impl ResourceDiscovery for CommandDiscovery { } // helper to insert a resource into a vector of resources in order of newest to oldest +// TODO: This should be a BTreeMap of the resource name and a BTreeMap of the version and DscResource, this keeps it version sorted more efficiently fn insert_resource(resources: &mut BTreeMap>, resource: &DscResource, skip_duplicate_version: bool) { if resources.contains_key(&resource.type_name) { let Some(resource_versions) = resources.get_mut(&resource.type_name) else { @@ -499,26 +548,39 @@ fn insert_resource(resources: &mut BTreeMap>, resource: } } -fn load_manifest(path: &Path) -> Result { - let file = File::open(path)?; - let reader = BufReader::new(file); - let manifest: ResourceManifest = if path.extension() == Some(OsStr::new("json")) { - match serde_json::from_reader(reader) { +fn load_manifest(path: &Path) -> Result { + let contents = fs::read_to_string(path)?; + if path.extension() == Some(OsStr::new("json")) { + if let Ok(manifest) = serde_json::from_str::(&contents) { + let resource = load_resource_manifest(path, &manifest)?; + return Ok(ManifestResource::Resource(resource)); + } + let manifest = match serde_json::from_str::(&contents) { Ok(manifest) => manifest, Err(err) => { - return Err(DscError::Manifest(path.to_string_lossy().to_string(), err)); + return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}"))); } - } + }; + let extension = load_extension_manifest(path, &manifest)?; + return Ok(ManifestResource::Extension(extension)); } else { - match serde_yaml::from_reader(reader) { + if let Ok(manifest) = serde_yaml::from_str::(&contents) { + let resource = load_resource_manifest(path, &manifest)?; + return Ok(ManifestResource::Resource(resource)); + } + let manifest = match serde_yaml::from_str::(&contents) { Ok(manifest) => manifest, Err(err) => { - return Err(DscError::ManifestYaml(path.to_string_lossy().to_string(), err)); + return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}"))); } - } - }; + }; + let extension = load_extension_manifest(path, &manifest)?; + return Ok(ManifestResource::Extension(extension)); + } +} +fn load_resource_manifest(path: &Path, manifest: &ResourceManifest) -> Result { if let Err(err) = validate_semver(&manifest.version) { return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}"))); } @@ -583,6 +645,31 @@ fn load_manifest(path: &Path) -> Result { Ok(resource) } +fn load_extension_manifest(path: &Path, manifest: &ExtensionManifest) -> Result { + if let Err(err) = validate_semver(&manifest.version) { + return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}"))); + } + + let mut capabilities: Vec = vec![]; + if let Some(discover) = &manifest.discover { + verify_executable(&manifest.r#type, "discover", &discover.executable); + capabilities.push(dscextension::Capability::Discover); + } + + let extension = DscExtension { + type_name: manifest.r#type.clone(), + description: manifest.description.clone(), + version: manifest.version.clone(), + capabilities, + path: path.to_str().unwrap().to_string(), + directory: path.parent().unwrap().to_str().unwrap().to_string(), + manifest: Some(serde_json::to_value(manifest)?), + ..Default::default() + }; + + Ok(extension) +} + fn verify_executable(resource: &str, operation: &str, executable: &str) { if which(executable).is_err() { warn!("{}", t!("discovery.commandDiscovery.executableNotFound", resource = resource, operation = operation, executable = executable)); diff --git a/dsc_lib/src/discovery/discovery_trait.rs b/dsc_lib/src/discovery/discovery_trait.rs index 1c652bf7..32842da5 100644 --- a/dsc_lib/src/discovery/discovery_trait.rs +++ b/dsc_lib/src/discovery/discovery_trait.rs @@ -1,17 +1,20 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::{dscresources::dscresource::DscResource, dscerror::DscError}; +use crate::{dscerror::DscError, dscresources::dscresource::DscResource}; use std::collections::BTreeMap; +use super::command_discovery::ManifestResource; + +#[derive(PartialEq)] pub enum DiscoveryKind { Resource, Extension, } pub trait ResourceDiscovery { - fn discover_resources(&mut self, filter: &str) -> Result<(), DscError>; + fn discover(&mut self, kind: &DiscoveryKind, filter: &str) -> Result<(), DscError>; fn discover_adapted_resources(&mut self, name_filter: &str, adapter_filter: &str) -> Result<(), DscError>; - fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError>; + fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError>; fn find_resources(&mut self, required_resource_types: &[String]) -> Result, DscError>; } diff --git a/dsc_lib/src/discovery/mod.rs b/dsc_lib/src/discovery/mod.rs index edcc29ba..c3dba063 100644 --- a/dsc_lib/src/discovery/mod.rs +++ b/dsc_lib/src/discovery/mod.rs @@ -1,17 +1,20 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -mod command_discovery; -mod discovery_trait; +pub mod command_discovery; +pub mod discovery_trait; -use crate::discovery::discovery_trait::ResourceDiscovery; +use crate::discovery::discovery_trait::{DiscoveryKind, ResourceDiscovery}; +use crate::extensions::dscextension::DscExtension; use crate::{dscresources::dscresource::DscResource, dscerror::DscError, progress::ProgressFormat}; use std::collections::BTreeMap; -use tracing::error; +use command_discovery::ManifestResource; +use tracing::{debug, error}; #[derive(Clone)] pub struct Discovery { pub resources: BTreeMap, + pub extensions: BTreeMap, } impl Discovery { @@ -24,6 +27,7 @@ impl Discovery { pub fn new() -> Result { Ok(Self { resources: BTreeMap::new(), + extensions: BTreeMap::new(), }) } @@ -31,22 +35,23 @@ impl Discovery { /// /// # Arguments /// + /// * `kind` - The kind of discovery (e.g., Resource). /// * `type_name_filter` - The filter for the resource type name. /// * `adapter_name_filter` - The filter for the adapter name. /// /// # Returns /// /// A vector of `DscResource` instances. - pub fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str, progress_format: ProgressFormat) -> Vec { + pub fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str, progress_format: ProgressFormat) -> Vec { let discovery_types: Vec> = vec![ Box::new(command_discovery::CommandDiscovery::new(progress_format)), ]; - let mut resources: Vec = Vec::new(); + let mut resources: Vec = Vec::new(); for mut discovery_type in discovery_types { - let discovered_resources = match discovery_type.list_available_resources(type_name_filter, adapter_name_filter) { + let discovered_resources = match discovery_type.list_available(kind, type_name_filter, adapter_name_filter) { Ok(value) => value, Err(err) => { error!("{err}"); diff --git a/dsc_lib/src/extensions/discover.rs b/dsc_lib/src/extensions/discover.rs new file mode 100644 index 00000000..4955870d --- /dev/null +++ b/dsc_lib/src/extensions/discover.rs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::dscresources::resource_manifest::ArgKind; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize, JsonSchema)] +pub struct DiscoverMethod { + /// The command to run to get the state of the resource. + pub executable: String, + /// The arguments to pass to the command to perform a Get. + pub args: Option>, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema)] +pub struct DiscoverResult { + #[serde(rename = "resourceManifestPath")] + pub resource_manifest_path: String, +} diff --git a/dsc_lib/src/extensions/dscextension.rs b/dsc_lib/src/extensions/dscextension.rs new file mode 100644 index 00000000..cd197e36 --- /dev/null +++ b/dsc_lib/src/extensions/dscextension.rs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use schemars::JsonSchema; + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DscExtension { + /// The namespaced name of the resource. + #[serde(rename="type")] + pub type_name: String, + /// The version of the resource. + pub version: String, + /// The capabilities of the resource. + pub capabilities: Vec, + /// The file path to the resource. + pub path: String, + /// The description of the resource. + pub description: Option, + // The directory path to the resource. + pub directory: String, + /// The author of the resource. + pub author: Option, + /// The manifest of the resource. + pub manifest: Option, +} + +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum Capability { + /// The extension aids in discovering resources. + Discover, +} + +impl DscExtension { + #[must_use] + pub fn new() -> Self { + Self { + type_name: String::new(), + version: String::new(), + capabilities: Vec::new(), + description: None, + path: String::new(), + directory: String::new(), + author: None, + manifest: None, + } + } +} + +impl Default for DscExtension { + fn default() -> Self { + DscExtension::new() + } +} \ No newline at end of file diff --git a/dsc_lib/src/extensions/extension_manifest.rs b/dsc_lib/src/extensions/extension_manifest.rs index 44eabb3d..b36675d3 100644 --- a/dsc_lib/src/extensions/extension_manifest.rs +++ b/dsc_lib/src/extensions/extension_manifest.rs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::dscresources::resource_manifest::ArgKind; use rust_i18n::t; use schemars::JsonSchema; use semver::Version; @@ -10,6 +9,7 @@ use serde_json::Value; use std::collections::HashMap; use crate::{dscerror::DscError, schemas::DscRepoSchema}; +use crate::extensions::discover::DiscoverMethod; #[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -34,14 +34,6 @@ pub struct ExtensionManifest { pub exit_codes: Option>, } -#[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize, JsonSchema)] -pub struct DiscoverMethod { - /// The command to run to get the state of the resource. - pub executable: String, - /// The arguments to pass to the command to perform a Get. - pub args: Option>, -} - impl DscRepoSchema for ExtensionManifest { const SCHEMA_FILE_BASE_NAME: &'static str = "manifest"; const SCHEMA_FOLDER_PATH: &'static str = "extension"; diff --git a/dsc_lib/src/extensions/mod.rs b/dsc_lib/src/extensions/mod.rs new file mode 100644 index 00000000..685c7af6 --- /dev/null +++ b/dsc_lib/src/extensions/mod.rs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +pub mod discover; +pub mod dscextension; +pub mod extension_manifest; diff --git a/dsc_lib/src/lib.rs b/dsc_lib/src/lib.rs index 2e9f49e0..e0ffc313 100644 --- a/dsc_lib/src/lib.rs +++ b/dsc_lib/src/lib.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +use crate::discovery::command_discovery::ManifestResource; +use crate::discovery::discovery_trait::DiscoveryKind; use crate::progress::ProgressFormat; use configure::config_doc::ExecutionKind; @@ -12,6 +14,7 @@ pub mod configure; pub mod discovery; pub mod dscerror; pub mod dscresources; +pub mod extensions; pub mod functions; pub mod parser; pub mod progress; @@ -48,8 +51,8 @@ impl DscManager { self.discovery.find_resource(name) } - pub fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str, progress_format: ProgressFormat) -> Vec { - self.discovery.list_available_resources(type_name_filter, adapter_name_filter, progress_format) + pub fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str, progress_format: ProgressFormat) -> Vec { + self.discovery.list_available(kind, type_name_filter, adapter_name_filter, progress_format) } pub fn find_resources(&mut self, required_resource_types: &[String], progress_format: ProgressFormat) { From 01862376f240b6868a483ac4bc6b02521e0d14db Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Wed, 23 Apr 2025 17:39:39 -0700 Subject: [PATCH 04/10] add execution of extension during discovery --- dsc/src/subcommand.rs | 2 +- dsc/tests/dsc_extension_discover.ps1 | 24 ++++ dsc_lib/locales/en-us.toml | 10 ++ dsc_lib/src/discovery/command_discovery.rs | 105 +++++++++++++----- dsc_lib/src/discovery/discovery_trait.rs | 60 ++++++++++ dsc_lib/src/discovery/mod.rs | 2 +- dsc_lib/src/dscerror.rs | 3 + dsc_lib/src/dscresources/command_resource.rs | 12 +- dsc_lib/src/extensions/dscextension.rs | 72 +++++++++++- extensions/test/discover/discover.ps1 | 9 ++ .../testDiscoveredOne.dsc.resource.json | 30 +++++ .../testDiscoveredTwo.dsc.resource.json | 30 +++++ .../discover/testDiscover.dsc.extension.json | 16 +++ 13 files changed, 339 insertions(+), 36 deletions(-) create mode 100644 dsc/tests/dsc_extension_discover.ps1 create mode 100644 extensions/test/discover/discover.ps1 create mode 100644 extensions/test/discover/resources/testDiscoveredOne.dsc.resource.json create mode 100644 extensions/test/discover/resources/testDiscoveredTwo.dsc.resource.json create mode 100644 extensions/test/discover/testDiscover.dsc.extension.json diff --git a/dsc/src/subcommand.rs b/dsc/src/subcommand.rs index a1a182de..cda5a18f 100644 --- a/dsc/src/subcommand.rs +++ b/dsc/src/subcommand.rs @@ -624,7 +624,7 @@ fn list_extensions(dsc: &mut DscManager, extension_name: Option<&String>, format write_table = true; } let mut include_separator = false; - for manifest_resource in dsc.list_available(&DiscoveryKind::Extension, extension_name.unwrap_or(&String::from("*")), &String::new(), progress_format) { + for manifest_resource in dsc.list_available(&DiscoveryKind::Extension, extension_name.unwrap_or(&String::from("*")), "", progress_format) { if let ManifestResource::Extension(extension) = manifest_resource { let mut capabilities = "-".to_string(); let capability_types = [ diff --git a/dsc/tests/dsc_extension_discover.ps1 b/dsc/tests/dsc_extension_discover.ps1 new file mode 100644 index 00000000..1178c9cc --- /dev/null +++ b/dsc/tests/dsc_extension_discover.ps1 @@ -0,0 +1,24 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +Describe 'Discover extension tests' { + BeforeAll { + $oldPath = $env:PATH + $separator = [System.IO.Path]::PathSeparator + $env:PATH = "$PSScriptRoot$separator$oldPath" + } + + AfterAll { + $env:PATH = $oldPath + } + + It 'Discover extensions' { + $out = dsc extension list | ConvertFrom-Json + $LASTEXITCODE | Should -Be 0 + $out.Count | Should -Be 1 + $out.type | Should -BeExactly 'Test/Discover' + $out.version | Should -BeExactly '0.1.0' + $out.capabilities | Should -BeExactly @('discover') + $out.manifest | Should -Not -BeNullOrEmpty + } +} diff --git a/dsc_lib/locales/en-us.toml b/dsc_lib/locales/en-us.toml index 1ff30d77..d43a7c33 100644 --- a/dsc_lib/locales/en-us.toml +++ b/dsc_lib/locales/en-us.toml @@ -81,12 +81,17 @@ usingResourcePath = "Using Resource Path: %{path}" discoverResources = "Discovering resources using filter: %{filter}" invalidAdapterFilter = "Could not build Regex filter for adapter name" progressSearching = "Searching for resources" +extensionSearching = "Searching for extensions" foundResourceManifest = "Found resource manifest: %{path}" extensionFound = "Extension '%{extension}' found" adapterFound = "Resource adapter '%{adapter}' found" resourceFound = "Resource '%{resource}' found" executableNotFound = "Executable '%{executable}' not found for operation '%{operation}' for resource '%{resource}'" extensionInvalidVersion = "Extension '%{extension}' version '%{version}' is invalid" +invalidManifest = "Invalid manifest for resource '%{resource}'" +extensionResourceFound = "Extension found resource '%{resource}'" +callingExtension = "Calling extension '%{extension}' to discover resources" +extensionFoundResources = "Extension '%{extension}' found %{count} resources" [dscresources.commandResource] invokeGet = "Invoking get for '%{resource}'" @@ -165,6 +170,9 @@ diffMissingItem = "diff: actual array missing expected item" resourceManifestSchemaTitle = "Resource manifest schema URI" resourceManifestSchemaDescription = "Defines the JSON Schema the resource manifest adheres to." +[extensions.dscextension] +discoverNoResults = "No results returned for discovery extension '%{extension}'" + [extensions.extension_manifest] extensionManifestSchemaTitle = "Extension manifest schema URI" extensionManifestSchemaDescription = "Defines the JSON Schema the extension manifest adheres to." @@ -342,6 +350,8 @@ unknown = "Unknown" unrecognizedSchemaUri = "Unrecognized $schema URI" validation = "Validation" validSchemaUrisAre = "Valid schema URIs are" +extension = "Extension" +unsupportedCapability = "does not support capability" setting = "Setting" [progress] diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 9e95a2a1..1ca60f7e 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -7,7 +7,7 @@ use crate::dscresources::dscresource::{Capability, DscResource, ImplementedAs}; use crate::dscresources::resource_manifest::{import_manifest, validate_semver, Kind, ResourceManifest, SchemaKind}; use crate::dscresources::command_resource::invoke_command; use crate::dscerror::DscError; -use crate::extensions::dscextension::{self, DscExtension}; +use crate::extensions::dscextension::{self, DscExtension, Capability as ExtensionCapability}; use crate::extensions::extension_manifest::ExtensionManifest; use crate::progress::{ProgressBar, ProgressFormat}; use linked_hash_map::LinkedHashMap; @@ -68,6 +68,7 @@ impl Default for ResourcePathSetting { } impl CommandDiscovery { + #[must_use] pub fn new(progress_format: ProgressFormat) -> CommandDiscovery { CommandDiscovery { adapters: BTreeMap::new(), @@ -184,6 +185,11 @@ impl ResourceDiscovery for CommandDiscovery { fn discover(&mut self, kind: &DiscoveryKind, filter: &str) -> Result<(), DscError> { info!("{}", t!("discovery.commandDiscovery.discoverResources", filter = filter)); + // if kind is DscResource, we need to discover extensions first + if *kind == DiscoveryKind::Resource { + self.discover(&DiscoveryKind::Extension, "*")?; + } + let regex_str = convert_wildcard_to_regex(filter); debug!("Using regex {regex_str} as filter for adapter name"); let mut regex_builder = RegexBuilder::new(®ex_str); @@ -193,7 +199,14 @@ impl ResourceDiscovery for CommandDiscovery { }; let mut progress = ProgressBar::new(1, self.progress_format)?; - progress.write_activity(t!("discovery.commandDiscovery.progressSearching").to_string().as_str()); + match kind { + DiscoveryKind::Resource => { + progress.write_activity(t!("discovery.commandDiscovery.progressSearching").to_string().as_str()); + }, + DiscoveryKind::Extension => { + progress.write_activity(t!("discovery.commandDiscovery.extensionSearching").to_string().as_str()); + } + } let mut adapters = BTreeMap::>::new(); let mut resources = BTreeMap::>::new(); @@ -217,7 +230,7 @@ impl ResourceDiscovery for CommandDiscovery { let file_name_lowercase = file_name.to_lowercase(); if (kind == &DiscoveryKind::Resource && DSC_RESOURCE_EXTENSIONS.iter().any(|ext| file_name_lowercase.ends_with(ext))) || (kind == &DiscoveryKind::Extension && DSC_EXTENSION_EXTENSIONS.iter().any(|ext| file_name_lowercase.ends_with(ext))) { - trace!("{}", t!("discovery.commandDiscovery.foundManifest", path = path.to_string_lossy())); + trace!("{}", t!("discovery.commandDiscovery.foundResourceManifest", path = path.to_string_lossy())); let resource = match load_manifest(&path) { Ok(r) => r, @@ -274,10 +287,31 @@ impl ResourceDiscovery for CommandDiscovery { } progress.write_increment(1); - debug!("Found {} matching non-adapter-based resources", resources.len()); - self.adapters = adapters; - self.resources = resources; - self.extensions = extensions; + + match kind { + DiscoveryKind::Resource => { + // Now we need to call discover extensions and add those resource to the list of resources + for (_extension_name, extension) in self.extensions.iter() { + if extension.capabilities.contains(&ExtensionCapability::Discover) { + debug!("{}", t!("discovery.commandDiscovery.callingExtension", extension = extension.type_name)); + let discovered_resources = extension.discover()?; + debug!("{}", t!("discovery.commandDiscovery.extensionFoundResources", extension = extension.type_name, count = discovered_resources.len())); + for resource in discovered_resources { + if regex.is_match(&resource.type_name) { + trace!("{}", t!("discovery.commandDiscovery.extensionResourceFound", resource = resource.type_name)); + insert_resource(&mut resources, &resource, true); + } + } + } + } + self.adapters = adapters; + self.resources = resources; + }, + DiscoveryKind::Extension => { + self.extensions = extensions; + } + } + Ok(()) } @@ -505,7 +539,6 @@ impl ResourceDiscovery for CommandDiscovery { } } -// helper to insert a resource into a vector of resources in order of newest to oldest // TODO: This should be a BTreeMap of the resource name and a BTreeMap of the version and DscResource, this keeps it version sorted more efficiently fn insert_resource(resources: &mut BTreeMap>, resource: &DscResource, skip_duplicate_version: bool) { if resources.contains_key(&resource.type_name) { @@ -548,36 +581,48 @@ fn insert_resource(resources: &mut BTreeMap>, resource: } } -fn load_manifest(path: &Path) -> Result { +/// Loads a manifest from the given path and returns a `ManifestResource`. +/// +/// # Arguments +/// +/// * `path` - The path to the manifest file. +/// +/// # Returns +/// +/// * `ManifestResource` if the manifest was loaded successfully. +/// +/// # Errors +/// +/// * Returns a `DscError` if the manifest could not be loaded or parsed. +pub fn load_manifest(path: &Path) -> Result { let contents = fs::read_to_string(path)?; if path.extension() == Some(OsStr::new("json")) { - if let Ok(manifest) = serde_json::from_str::(&contents) { - let resource = load_resource_manifest(path, &manifest)?; - return Ok(ManifestResource::Resource(resource)); + if let Ok(manifest) = serde_json::from_str::(&contents) { + let extension = load_extension_manifest(path, &manifest)?; + return Ok(ManifestResource::Extension(extension)); } - let manifest = match serde_json::from_str::(&contents) { + let manifest = match serde_json::from_str::(&contents) { Ok(manifest) => manifest, Err(err) => { - return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}"))); + return Err(DscError::Manifest(t!("discovery.commandDiscovery.invalidManifest", resource = path.to_string_lossy()).to_string(), err)); } }; - let extension = load_extension_manifest(path, &manifest)?; - return Ok(ManifestResource::Extension(extension)); + let resource = load_resource_manifest(path, &manifest)?; + return Ok(ManifestResource::Resource(resource)); } - else { - if let Ok(manifest) = serde_yaml::from_str::(&contents) { - let resource = load_resource_manifest(path, &manifest)?; - return Ok(ManifestResource::Resource(resource)); - } - let manifest = match serde_yaml::from_str::(&contents) { - Ok(manifest) => manifest, - Err(err) => { - return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}"))); - } - }; - let extension = load_extension_manifest(path, &manifest)?; - return Ok(ManifestResource::Extension(extension)); + + if let Ok(manifest) = serde_yaml::from_str::(&contents) { + let resource = load_resource_manifest(path, &manifest)?; + return Ok(ManifestResource::Resource(resource)); } + let manifest = match serde_yaml::from_str::(&contents) { + Ok(manifest) => manifest, + Err(err) => { + return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}"))); + } + }; + let extension = load_extension_manifest(path, &manifest)?; + Ok(ManifestResource::Extension(extension)) } fn load_resource_manifest(path: &Path, manifest: &ResourceManifest) -> Result { @@ -663,7 +708,7 @@ fn load_extension_manifest(path: &Path, manifest: &ExtensionManifest) -> Result< capabilities, path: path.to_str().unwrap().to_string(), directory: path.parent().unwrap().to_str().unwrap().to_string(), - manifest: Some(serde_json::to_value(manifest)?), + manifest: serde_json::to_value(manifest)?, ..Default::default() }; diff --git a/dsc_lib/src/discovery/discovery_trait.rs b/dsc_lib/src/discovery/discovery_trait.rs index 32842da5..4a6f8c87 100644 --- a/dsc_lib/src/discovery/discovery_trait.rs +++ b/dsc_lib/src/discovery/discovery_trait.rs @@ -13,8 +13,68 @@ pub enum DiscoveryKind { } pub trait ResourceDiscovery { + /// Discovery method to find resources. + /// + /// # Arguments + /// + /// * `kind` - The kind of discovery (e.g., Resource). + /// * `filter` - The filter for the resource type name. + /// + /// # Returns + /// + /// A result indicating success or failure. + /// + /// # Errors + /// + /// This function will return an error if the underlying discovery fails. fn discover(&mut self, kind: &DiscoveryKind, filter: &str) -> Result<(), DscError>; + + /// Discover adapted resources based on the provided filters. + /// + /// # Arguments + /// + /// * `name_filter` - The filter for the resource name. + /// * `adapter_filter` - The filter for the adapter name. + /// + /// # Returns + /// + /// A result indicating success or failure. + /// + /// # Errors + /// + /// This function will return an error if the underlying discovery fails. fn discover_adapted_resources(&mut self, name_filter: &str, adapter_filter: &str) -> Result<(), DscError>; + + /// List available resources based on the provided filters. + /// + /// # Arguments + /// + /// * `kind` - The kind of discovery (e.g., Resource). + /// * `type_name_filter` - The filter for the resource type name. + /// * `adapter_name_filter` - The filter for the adapter name (only applies to resources). + /// + /// # Returns + /// + /// A result containing a map of resource names to their corresponding `ManifestResource` instances. + /// + /// # Errors + /// + /// This function will return an error if the underlying discovery fails. fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError>; + + /// Find resources based on the required resource types. + /// This is not applicable for extensions. + /// + /// # Arguments + /// + /// * `required_resource_types` - A slice of strings representing the required resource types. + /// + /// # Returns + /// + /// A result containing a map of resource names to their corresponding `DscResource` instances. + /// + /// # Errors + /// + /// This function will return an error if the underlying discovery fails. fn find_resources(&mut self, required_resource_types: &[String]) -> Result, DscError>; } diff --git a/dsc_lib/src/discovery/mod.rs b/dsc_lib/src/discovery/mod.rs index c3dba063..d81f3956 100644 --- a/dsc_lib/src/discovery/mod.rs +++ b/dsc_lib/src/discovery/mod.rs @@ -9,7 +9,7 @@ use crate::extensions::dscextension::DscExtension; use crate::{dscresources::dscresource::DscResource, dscerror::DscError, progress::ProgressFormat}; use std::collections::BTreeMap; use command_discovery::ManifestResource; -use tracing::{debug, error}; +use tracing::error; #[derive(Clone)] pub struct Discovery { diff --git a/dsc_lib/src/dscerror.rs b/dsc_lib/src/dscerror.rs index 4437aeb4..06af63b3 100644 --- a/dsc_lib/src/dscerror.rs +++ b/dsc_lib/src/dscerror.rs @@ -116,6 +116,9 @@ pub enum DscError { #[error("{t}: {0}. {t2}: {1:?}", t = t!("dscerror.unrecognizedSchemaUri"), t2 = t!("dscerror.validSchemaUrisAre"))] UnrecognizedSchemaUri(String, Vec), + #[error("{t} '{0}' {t2} '{1}'", t = t!("dscerror.extension"), t2 = t!("dscerror.unsupportedCapability"))] + UnsupportedCapability(String, String), + #[error("{t}: {0}", t = t!("dscerror.validation"))] Validation(String), diff --git a/dsc_lib/src/dscresources/command_resource.rs b/dsc_lib/src/dscresources/command_resource.rs index 83474eee..41b69e15 100644 --- a/dsc_lib/src/dscresources/command_resource.rs +++ b/dsc_lib/src/dscresources/command_resource.rs @@ -705,7 +705,17 @@ pub fn invoke_command(executable: &str, args: Option>, input: Option .block_on(run_process_async(executable, args, input, cwd, env, exit_codes)) } -fn process_args(args: Option<&Vec>, value: &str) -> Option> { +/// Process the arguments for a command resource. +/// +/// # Arguments +/// +/// * `args` - The arguments to process +/// * `value` - The value to use for JSON input arguments +/// +/// # Returns +/// +/// A vector of strings representing the processed arguments +pub fn process_args(args: Option<&Vec>, value: &str) -> Option> { let Some(arg_values) = args else { debug!("{}", t!("dscresources.commandResource.noArgs")); return None; diff --git a/dsc_lib/src/extensions/dscextension.rs b/dsc_lib/src/extensions/dscextension.rs index cd197e36..ff9d6185 100644 --- a/dsc_lib/src/extensions/dscextension.rs +++ b/dsc_lib/src/extensions/dscextension.rs @@ -1,9 +1,16 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +use rust_i18n::t; use serde::{Deserialize, Serialize}; use serde_json::Value; use schemars::JsonSchema; +use std::{fmt::Display, path::Path}; +use tracing::info; + +use crate::{discovery::command_discovery::{load_manifest, ManifestResource}, dscerror::DscError, dscresources::{command_resource::{invoke_command, process_args}, dscresource::DscResource}}; + +use super::{discover::DiscoverResult, extension_manifest::ExtensionManifest}; #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -24,7 +31,7 @@ pub struct DscExtension { /// The author of the resource. pub author: Option, /// The manifest of the resource. - pub manifest: Option, + pub manifest: Value, } #[derive(Clone, Debug, PartialEq, Deserialize, Serialize, JsonSchema)] @@ -34,6 +41,14 @@ pub enum Capability { Discover, } +impl Display for Capability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Capability::Discover => write!(f, "Discover"), + } + } +} + impl DscExtension { #[must_use] pub fn new() -> Self { @@ -45,7 +60,58 @@ impl DscExtension { path: String::new(), directory: String::new(), author: None, - manifest: None, + manifest: Value::Null, + } + } + + pub fn discover(&self) -> Result, DscError> { + let mut resources: Vec = Vec::new(); + + if self.capabilities.contains(&Capability::Discover) { + let extension = match serde_json::from_value::(self.manifest.clone()) { + Ok(manifest) => manifest, + Err(err) => { + return Err(DscError::Manifest(self.type_name.clone(), err)); + } + }; + let Some(discover) = extension.discover else { + return Err(DscError::UnsupportedCapability(self.type_name.clone(), Capability::Discover.to_string())); + }; + let args = process_args(discover.args.as_ref(), ""); + let (_exit_code, stdout, _stderr) = invoke_command( + &discover.executable, + args, + None, + Some(self.directory.as_str()), + None, + extension.exit_codes.as_ref(), + )?; + if stdout.is_empty() { + info!("{}", t!("extensions.dscextension.discoverNoResults", extension = self.type_name)); + } else { + for line in stdout.lines() { + let discover_result: DiscoverResult = match serde_json::from_str(line) { + Ok(result) => result, + Err(err) => { + return Err(DscError::Json(err)); + } + }; + let manifest_path = Path::new(&discover_result.resource_manifest_path); + if let ManifestResource::Resource(resource) = load_manifest(manifest_path)? { + resources.push(resource); + } else { + // ignore loading other types of manifests + continue; + } + } + } + + Ok(resources) + } else { + Err(DscError::UnsupportedCapability( + self.type_name.clone(), + Capability::Discover.to_string() + )) } } } @@ -54,4 +120,4 @@ impl Default for DscExtension { fn default() -> Self { DscExtension::new() } -} \ No newline at end of file +} diff --git a/extensions/test/discover/discover.ps1 b/extensions/test/discover/discover.ps1 new file mode 100644 index 00000000..9af4383a --- /dev/null +++ b/extensions/test/discover/discover.ps1 @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +Get-ChildItem -Path $PSScriptRoot/resources/*.json | ForEach-Object { + $resource = [pscustomobject]@{ + resourceManifestPath = $_.FullName + } + $resource | ConvertTo-Json -Compress +} diff --git a/extensions/test/discover/resources/testDiscoveredOne.dsc.resource.json b/extensions/test/discover/resources/testDiscoveredOne.dsc.resource.json new file mode 100644 index 00000000..b0b02041 --- /dev/null +++ b/extensions/test/discover/resources/testDiscoveredOne.dsc.resource.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json", + "type": "Test/DiscoveredOne", + "version": "0.1.0", + "description": "First discovered resource", + "get": { + "executable": "pwsh", + "args": [ + "-NoLogo", + "-NonInteractive", + "-NoProfile", + "-Command", + "'{\"Output\": \"One\"}'" + ] + }, + "schema": { + "embedded": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://raw.githubusercontent.com/PowerShell/DSC/main/schemas/resources/Test/DiscoveredOne/v0.1.0/schema.json", + "title": "TestDiscoveredOne", + "description": "First discovered resource", + "type": "object", + "required": [], + "additionalProperties": false + } + }, + "exitCodes": { + "0": "Success" + } +} diff --git a/extensions/test/discover/resources/testDiscoveredTwo.dsc.resource.json b/extensions/test/discover/resources/testDiscoveredTwo.dsc.resource.json new file mode 100644 index 00000000..229a8e97 --- /dev/null +++ b/extensions/test/discover/resources/testDiscoveredTwo.dsc.resource.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json", + "type": "Test/DiscoveredTwo", + "version": "0.1.0", + "description": "Second discovered resource", + "get": { + "executable": "pwsh", + "args": [ + "-NoLogo", + "-NonInteractive", + "-NoProfile", + "-Command", + "'{\"Output\": \"Two\"}'" + ] + }, + "schema": { + "embedded": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://raw.githubusercontent.com/PowerShell/DSC/main/schemas/resources/Test/DiscoveredTwo/v0.1.0/schema.json", + "title": "TestDiscoveredTwo", + "description": "Second discovered resource", + "type": "object", + "required": [], + "additionalProperties": false + } + }, + "exitCodes": { + "0": "Success" + } +} diff --git a/extensions/test/discover/testDiscover.dsc.extension.json b/extensions/test/discover/testDiscover.dsc.extension.json new file mode 100644 index 00000000..2a8db380 --- /dev/null +++ b/extensions/test/discover/testDiscover.dsc.extension.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json", + "type": "Test/Discover", + "version": "0.1.0", + "description": "Test discover resource", + "discover": { + "executable": "pwsh", + "args": [ + "-NoLogo", + "-NonInteractive", + "-NoProfile", + "-Command", + "./discover.ps1" + ] + } +} From d062cfe445c5104ff225dd754acc2ac5b4094c3b Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Wed, 23 Apr 2025 18:01:27 -0700 Subject: [PATCH 05/10] add tests --- dsc/tests/dsc_extension_discover.ps1 | 29 +++++++++++++++++++ .../testDiscoveredOne.dsc.resource.json | 2 +- .../testDiscoveredTwo.dsc.resource.json | 2 +- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/dsc/tests/dsc_extension_discover.ps1 b/dsc/tests/dsc_extension_discover.ps1 index 1178c9cc..257ed308 100644 --- a/dsc/tests/dsc_extension_discover.ps1 +++ b/dsc/tests/dsc_extension_discover.ps1 @@ -21,4 +21,33 @@ Describe 'Discover extension tests' { $out.capabilities | Should -BeExactly @('discover') $out.manifest | Should -Not -BeNullOrEmpty } + + It 'Filtering works for extension discovered resources' { + $out = dsc resource list '*Discovered*' | ConvertFrom-Json + $LASTEXITCODE | Should -Be 0 + $out.Count | Should -Be 2 + $out[0].type | Should -Be 'Test/DiscoveredOne' + $out[1].type | Should -Be 'Test/DiscoveredTwo' + $out[0].kind | Should -Be 'Resource' + $out[1].kind | Should -Be 'Resource' + } + + It 'Extension resources can be used in config' { + $config_yaml = @" + `$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json + contentVersion: 1.0.0.0 + resources: + - name: One + type: Test/DiscoveredOne + - name: Two + type: Test/DiscoveredTwo +"@ + $out = dsc config get -i $config_yaml | ConvertFrom-Json + $LASTEXITCODE | Should -Be 0 + $out.results.Count | Should -Be 2 + $out.results[0].type | Should -BeExactly 'Test/DiscoveredOne' + $out.results[0].result.actualState.Output | Should -BeExactly 'Hello One' + $out.results[1].type | Should -BeExactly 'Test/DiscoveredTwo' + $out.results[1].result.actualState.Output | Should -BeExactly 'Hello Two' + } } diff --git a/extensions/test/discover/resources/testDiscoveredOne.dsc.resource.json b/extensions/test/discover/resources/testDiscoveredOne.dsc.resource.json index b0b02041..d564c733 100644 --- a/extensions/test/discover/resources/testDiscoveredOne.dsc.resource.json +++ b/extensions/test/discover/resources/testDiscoveredOne.dsc.resource.json @@ -10,7 +10,7 @@ "-NonInteractive", "-NoProfile", "-Command", - "'{\"Output\": \"One\"}'" + "'{\"Output\": \"Hello One\"}'" ] }, "schema": { diff --git a/extensions/test/discover/resources/testDiscoveredTwo.dsc.resource.json b/extensions/test/discover/resources/testDiscoveredTwo.dsc.resource.json index 229a8e97..60ddd2da 100644 --- a/extensions/test/discover/resources/testDiscoveredTwo.dsc.resource.json +++ b/extensions/test/discover/resources/testDiscoveredTwo.dsc.resource.json @@ -10,7 +10,7 @@ "-NonInteractive", "-NoProfile", "-Command", - "'{\"Output\": \"Two\"}'" + "'{\"Output\": \"Hello Two\"}'" ] }, "schema": { From 294f74acab70c9d5008653d8782c2e3bc5224450 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Wed, 23 Apr 2025 18:19:26 -0700 Subject: [PATCH 06/10] fix clippy --- dsc_lib/src/discovery/command_discovery.rs | 3 ++- dsc_lib/src/extensions/dscextension.rs | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 1ca60f7e..0cbd2488 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -182,6 +182,7 @@ impl Default for CommandDiscovery { impl ResourceDiscovery for CommandDiscovery { + #[allow(clippy::too_many_lines)] fn discover(&mut self, kind: &DiscoveryKind, filter: &str) -> Result<(), DscError> { info!("{}", t!("discovery.commandDiscovery.discoverResources", filter = filter)); @@ -291,7 +292,7 @@ impl ResourceDiscovery for CommandDiscovery { match kind { DiscoveryKind::Resource => { // Now we need to call discover extensions and add those resource to the list of resources - for (_extension_name, extension) in self.extensions.iter() { + for extension in self.extensions.values() { if extension.capabilities.contains(&ExtensionCapability::Discover) { debug!("{}", t!("discovery.commandDiscovery.callingExtension", extension = extension.type_name)); let discovered_resources = extension.discover()?; diff --git a/dsc_lib/src/extensions/dscextension.rs b/dsc_lib/src/extensions/dscextension.rs index ff9d6185..15a8b518 100644 --- a/dsc_lib/src/extensions/dscextension.rs +++ b/dsc_lib/src/extensions/dscextension.rs @@ -64,6 +64,15 @@ impl DscExtension { } } + /// Perform discovery of resources using the extension. + /// + /// # Returns + /// + /// A result containing a vector of discovered resources or an error. + /// + /// # Errors + /// + /// This function will return an error if the discovery fails. pub fn discover(&self) -> Result, DscError> { let mut resources: Vec = Vec::new(); @@ -97,11 +106,9 @@ impl DscExtension { } }; let manifest_path = Path::new(&discover_result.resource_manifest_path); + // Currently we don't support extensions discovering other extensions if let ManifestResource::Resource(resource) = load_manifest(manifest_path)? { resources.push(resource); - } else { - // ignore loading other types of manifests - continue; } } } From f879c08a7fd0d06eeff91993519dffa90b70eaae Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Wed, 23 Apr 2025 18:33:25 -0700 Subject: [PATCH 07/10] fix i18n --- dsc_lib/locales/en-us.toml | 3 --- dsc_lib/src/extensions/extension_manifest.rs | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dsc_lib/locales/en-us.toml b/dsc_lib/locales/en-us.toml index d43a7c33..6ff8b551 100644 --- a/dsc_lib/locales/en-us.toml +++ b/dsc_lib/locales/en-us.toml @@ -67,9 +67,6 @@ parameterNotObject = "Parameter '%{name}' is not an object" invokePropertyExpressions = "Invoke property expressions" invokeExpression = "Invoke property expression for %{name}: %{value}" -[discovery.mod] -extensionUnexpcted = "Extension '%{extension}' not expected" - [discovery.commandDiscovery] couldNotReadSetting = "Could not read 'resourcePath' setting" appendingEnvPath = "Appending PATH to resourcePath" diff --git a/dsc_lib/src/extensions/extension_manifest.rs b/dsc_lib/src/extensions/extension_manifest.rs index b36675d3..535a20cb 100644 --- a/dsc_lib/src/extensions/extension_manifest.rs +++ b/dsc_lib/src/extensions/extension_manifest.rs @@ -42,7 +42,7 @@ impl DscRepoSchema for ExtensionManifest { fn schema_metadata() -> schemars::schema::Metadata { schemars::schema::Metadata { title: Some(t!("extensions.extension_manifest.extensionManifestSchemaTitle").into()), - description: Some(t!("extensions.extension_manifest.extensioneManifestSchemaDescription").into()), + description: Some(t!("extensions.extension_manifest.extensionManifestSchemaDescription").into()), ..Default::default() } } From b7983381d3eac343905cefd7ac4c45416bbc7f94 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Thu, 24 Apr 2025 13:19:25 -0700 Subject: [PATCH 08/10] validate that the path is absolute --- dsc/tests/dsc_extension_discover.ps1 | 42 +++++++++++++++++++++++++- dsc_lib/locales/en-us.toml | 1 + dsc_lib/src/dscerror.rs | 3 ++ dsc_lib/src/extensions/dscextension.rs | 3 ++ extensions/test/discover/discover.ps1 | 12 +++++++- 5 files changed, 59 insertions(+), 2 deletions(-) diff --git a/dsc/tests/dsc_extension_discover.ps1 b/dsc/tests/dsc_extension_discover.ps1 index 257ed308..5428e3b9 100644 --- a/dsc/tests/dsc_extension_discover.ps1 +++ b/dsc/tests/dsc_extension_discover.ps1 @@ -5,7 +5,8 @@ Describe 'Discover extension tests' { BeforeAll { $oldPath = $env:PATH $separator = [System.IO.Path]::PathSeparator - $env:PATH = "$PSScriptRoot$separator$oldPath" + $toolPath = Resolve-Path -Path "$PSScriptRoot/../../extensions/test/discover" + $env:PATH = "$toolPath$separator$oldPath" } AfterAll { @@ -50,4 +51,43 @@ Describe 'Discover extension tests' { $out.results[1].type | Should -BeExactly 'Test/DiscoveredTwo' $out.results[1].result.actualState.Output | Should -BeExactly 'Hello Two' } + + It 'Relative path from discovery will fail' { + $extension_json = @' +{ + "$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json", + "type": "Test/DiscoverRelative", + "version": "0.1.0", + "description": "Test discover resource", + "discover": { + "executable": "pwsh", + "args": [ + "-NoLogo", + "-NonInteractive", + "-NoProfile", + "-Command", + "./discover.ps1", + "-RelativePath" + ] + } +} +'@ + Set-Content -Path "$TestDrive/test.dsc.extension.json" -Value $extension_json + Copy-Item -Path "$toolPath/discover.ps1" -Destination $TestDrive | Out-Null + Copy-Item -Path "$toolPath/resources" -Destination $TestDrive -Recurse | Out-Null + $env:DSC_RESOURCE_PATH = $TestDrive + try { + $out = dsc extension list | ConvertFrom-Json + $out.Count | Should -Be 1 + $out.type | Should -Be 'Test/DiscoverRelative' + $out = dsc resource list 2> $TestDrive/error.log + write-verbose -verbose (Get-Content -Path "$TestDrive/error.log" -Raw) + $LASTEXITCODE | Should -Be 0 + $out | Should -BeNullOrEmpty + $errorMessage = Get-Content -Path "$TestDrive/error.log" -Raw + $errorMessage | Should -BeLike '*is not an absolute path*' + } finally { + $env:DSC_RESOURCE_PATH = $null + } + } } diff --git a/dsc_lib/locales/en-us.toml b/dsc_lib/locales/en-us.toml index 6ff8b551..a140464c 100644 --- a/dsc_lib/locales/en-us.toml +++ b/dsc_lib/locales/en-us.toml @@ -169,6 +169,7 @@ resourceManifestSchemaDescription = "Defines the JSON Schema the resource manife [extensions.dscextension] discoverNoResults = "No results returned for discovery extension '%{extension}'" +discoverNotAbsolutePath = "Resource path from extension '%{extension}' is not an absolute path: %{path}" [extensions.extension_manifest] extensionManifestSchemaTitle = "Extension manifest schema URI" diff --git a/dsc_lib/src/dscerror.rs b/dsc_lib/src/dscerror.rs index 06af63b3..f9392301 100644 --- a/dsc_lib/src/dscerror.rs +++ b/dsc_lib/src/dscerror.rs @@ -29,6 +29,9 @@ pub enum DscError { #[error("{t} {0} {t2} '{1}'", t = t!("dscerror.commandOperation"), t2 = t!("dscerror.forExecutable"))] CommandOperation(String, String), + #[error("{0}")] + Extension(String), + #[error("{t} '{0}' {t2}: {1}", t = t!("dscerror.function"), t2 = t!("dscerror.error"))] Function(String, String), diff --git a/dsc_lib/src/extensions/dscextension.rs b/dsc_lib/src/extensions/dscextension.rs index 15a8b518..b6e4c8f2 100644 --- a/dsc_lib/src/extensions/dscextension.rs +++ b/dsc_lib/src/extensions/dscextension.rs @@ -105,6 +105,9 @@ impl DscExtension { return Err(DscError::Json(err)); } }; + if !Path::new(&discover_result.resource_manifest_path).is_absolute() { + return Err(DscError::Extension(t!("extensions.dscextension.discoverNotAbsolutePath", extension = self.type_name.clone(), path = discover_result.resource_manifest_path.clone()).to_string())); + } let manifest_path = Path::new(&discover_result.resource_manifest_path); // Currently we don't support extensions discovering other extensions if let ManifestResource::Resource(resource) = load_manifest(manifest_path)? { diff --git a/extensions/test/discover/discover.ps1 b/extensions/test/discover/discover.ps1 index 9af4383a..fbc8440f 100644 --- a/extensions/test/discover/discover.ps1 +++ b/extensions/test/discover/discover.ps1 @@ -1,9 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +[CmdletBinding()] +param( + [Parameter()] + [switch]$RelativePath +) + Get-ChildItem -Path $PSScriptRoot/resources/*.json | ForEach-Object { $resource = [pscustomobject]@{ - resourceManifestPath = $_.FullName + resourceManifestPath = if ($RelativePath) { + Resolve-Path -Path $_.FullName -Relative + } else { + $_.FullName + } } $resource | ConvertTo-Json -Compress } From 5cc664c6ceb64793d58f3c53b1e75abd9611de70 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Thu, 1 May 2025 14:20:32 -0700 Subject: [PATCH 09/10] address my own feedback --- ...c_extension_discover.ps1 => dsc_extension_discover.tests.ps1} | 0 dsc_lib/src/extensions/discover.rs | 1 + 2 files changed, 1 insertion(+) rename dsc/tests/{dsc_extension_discover.ps1 => dsc_extension_discover.tests.ps1} (100%) diff --git a/dsc/tests/dsc_extension_discover.ps1 b/dsc/tests/dsc_extension_discover.tests.ps1 similarity index 100% rename from dsc/tests/dsc_extension_discover.ps1 rename to dsc/tests/dsc_extension_discover.tests.ps1 diff --git a/dsc_lib/src/extensions/discover.rs b/dsc_lib/src/extensions/discover.rs index 4955870d..023b7968 100644 --- a/dsc_lib/src/extensions/discover.rs +++ b/dsc_lib/src/extensions/discover.rs @@ -15,6 +15,7 @@ pub struct DiscoverMethod { #[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema)] pub struct DiscoverResult { + /// The path to the resource manifest, must be absolute. #[serde(rename = "resourceManifestPath")] pub resource_manifest_path: String, } From 233c3f5d7777f5e19aabb52f71372fea66334654 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Mon, 5 May 2025 10:39:49 -0700 Subject: [PATCH 10/10] rename `ManifestResource` to `ImportedManifest` --- dsc/src/subcommand.rs | 6 ++--- dsc_lib/src/discovery/command_discovery.rs | 28 +++++++++++----------- dsc_lib/src/discovery/discovery_trait.rs | 4 ++-- dsc_lib/src/discovery/mod.rs | 6 ++--- dsc_lib/src/extensions/dscextension.rs | 4 ++-- dsc_lib/src/lib.rs | 4 ++-- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/dsc/src/subcommand.rs b/dsc/src/subcommand.rs index cda5a18f..c52a1ed2 100644 --- a/dsc/src/subcommand.rs +++ b/dsc/src/subcommand.rs @@ -17,7 +17,7 @@ use dsc_lib::{ Configurator, }, discovery::discovery_trait::DiscoveryKind, - discovery::command_discovery::ManifestResource, + discovery::command_discovery::ImportedManifest, dscerror::DscError, DscManager, dscresources::invoke_result::{ @@ -625,7 +625,7 @@ fn list_extensions(dsc: &mut DscManager, extension_name: Option<&String>, format } let mut include_separator = false; for manifest_resource in dsc.list_available(&DiscoveryKind::Extension, extension_name.unwrap_or(&String::from("*")), "", progress_format) { - if let ManifestResource::Extension(extension) = manifest_resource { + if let ImportedManifest::Extension(extension) = manifest_resource { let mut capabilities = "-".to_string(); let capability_types = [ (ExtensionCapability::Discover, "d"), @@ -683,7 +683,7 @@ fn list_resources(dsc: &mut DscManager, resource_name: Option<&String>, adapter_ } let mut include_separator = false; for manifest_resource in dsc.list_available(&DiscoveryKind::Resource, resource_name.unwrap_or(&String::from("*")), adapter_name.unwrap_or(&String::new()), progress_format) { - if let ManifestResource::Resource(resource) = manifest_resource { + if let ImportedManifest::Resource(resource) = manifest_resource { let mut capabilities = "--------".to_string(); let capability_types = [ (Capability::Get, "g"), diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 0cbd2488..42339ec2 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -31,7 +31,7 @@ const DSC_RESOURCE_EXTENSIONS: [&str; 3] = [".dsc.resource.json", ".dsc.resource const DSC_EXTENSION_EXTENSIONS: [&str; 3] = [".dsc.extension.json", ".dsc.extension.yaml", ".dsc.extension.yml"]; #[derive(Clone)] -pub enum ManifestResource { +pub enum ImportedManifest { Resource(DscResource), Extension(DscExtension), } @@ -246,7 +246,7 @@ impl ResourceDiscovery for CommandDiscovery { }; match resource { - ManifestResource::Extension(extension) => { + ImportedManifest::Extension(extension) => { if regex.is_match(&extension.type_name) { trace!("{}", t!("discovery.commandDiscovery.extensionFound", extension = extension.type_name)); // we only keep newest version of the extension so compare the version and only keep the newest @@ -265,7 +265,7 @@ impl ResourceDiscovery for CommandDiscovery { } } }, - ManifestResource::Resource(resource) => { + ImportedManifest::Resource(resource) => { if regex.is_match(&resource.type_name) { if let Some(ref manifest) = resource.manifest { let manifest = import_manifest(manifest.clone())?; @@ -423,19 +423,19 @@ impl ResourceDiscovery for CommandDiscovery { Ok(()) } - fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError> { + fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError> { trace!("Listing resources with type_name_filter '{type_name_filter}' and adapter_name_filter '{adapter_name_filter}'"); - let mut resources = BTreeMap::>::new(); + let mut resources = BTreeMap::>::new(); if *kind == DiscoveryKind::Resource { if adapter_name_filter.is_empty() { self.discover(kind, type_name_filter)?; for (resource_name, resources_vec) in &self.resources { - resources.insert(resource_name.clone(), resources_vec.iter().map(|r| ManifestResource::Resource(r.clone())).collect()); + resources.insert(resource_name.clone(), resources_vec.iter().map(|r| ImportedManifest::Resource(r.clone())).collect()); } for (adapter_name, adapter_vec) in &self.adapters { - resources.insert(adapter_name.clone(), adapter_vec.iter().map(|r| ManifestResource::Resource(r.clone())).collect()); + resources.insert(adapter_name.clone(), adapter_vec.iter().map(|r| ImportedManifest::Resource(r.clone())).collect()); } } else { self.discover(kind, "*")?; @@ -445,13 +445,13 @@ impl ResourceDiscovery for CommandDiscovery { add_resources_to_lookup_table(&self.adapted_resources); for (adapted_name, adapted_vec) in &self.adapted_resources { - resources.insert(adapted_name.clone(), adapted_vec.iter().map(|r| ManifestResource::Resource(r.clone())).collect()); + resources.insert(adapted_name.clone(), adapted_vec.iter().map(|r| ImportedManifest::Resource(r.clone())).collect()); } } } else { self.discover(kind, type_name_filter)?; for (extension_name, extension) in &self.extensions { - resources.insert(extension_name.clone(), vec![ManifestResource::Extension(extension.clone())]); + resources.insert(extension_name.clone(), vec![ImportedManifest::Extension(extension.clone())]); } } @@ -595,12 +595,12 @@ fn insert_resource(resources: &mut BTreeMap>, resource: /// # Errors /// /// * Returns a `DscError` if the manifest could not be loaded or parsed. -pub fn load_manifest(path: &Path) -> Result { +pub fn load_manifest(path: &Path) -> Result { let contents = fs::read_to_string(path)?; if path.extension() == Some(OsStr::new("json")) { if let Ok(manifest) = serde_json::from_str::(&contents) { let extension = load_extension_manifest(path, &manifest)?; - return Ok(ManifestResource::Extension(extension)); + return Ok(ImportedManifest::Extension(extension)); } let manifest = match serde_json::from_str::(&contents) { Ok(manifest) => manifest, @@ -609,12 +609,12 @@ pub fn load_manifest(path: &Path) -> Result { } }; let resource = load_resource_manifest(path, &manifest)?; - return Ok(ManifestResource::Resource(resource)); + return Ok(ImportedManifest::Resource(resource)); } if let Ok(manifest) = serde_yaml::from_str::(&contents) { let resource = load_resource_manifest(path, &manifest)?; - return Ok(ManifestResource::Resource(resource)); + return Ok(ImportedManifest::Resource(resource)); } let manifest = match serde_yaml::from_str::(&contents) { Ok(manifest) => manifest, @@ -623,7 +623,7 @@ pub fn load_manifest(path: &Path) -> Result { } }; let extension = load_extension_manifest(path, &manifest)?; - Ok(ManifestResource::Extension(extension)) + Ok(ImportedManifest::Extension(extension)) } fn load_resource_manifest(path: &Path, manifest: &ResourceManifest) -> Result { diff --git a/dsc_lib/src/discovery/discovery_trait.rs b/dsc_lib/src/discovery/discovery_trait.rs index 4a6f8c87..b6e61101 100644 --- a/dsc_lib/src/discovery/discovery_trait.rs +++ b/dsc_lib/src/discovery/discovery_trait.rs @@ -4,7 +4,7 @@ use crate::{dscerror::DscError, dscresources::dscresource::DscResource}; use std::collections::BTreeMap; -use super::command_discovery::ManifestResource; +use super::command_discovery::ImportedManifest; #[derive(PartialEq)] pub enum DiscoveryKind { @@ -60,7 +60,7 @@ pub trait ResourceDiscovery { /// # Errors /// /// This function will return an error if the underlying discovery fails. - fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError>; + fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError>; /// Find resources based on the required resource types. /// This is not applicable for extensions. diff --git a/dsc_lib/src/discovery/mod.rs b/dsc_lib/src/discovery/mod.rs index d81f3956..9e9c0622 100644 --- a/dsc_lib/src/discovery/mod.rs +++ b/dsc_lib/src/discovery/mod.rs @@ -8,7 +8,7 @@ use crate::discovery::discovery_trait::{DiscoveryKind, ResourceDiscovery}; use crate::extensions::dscextension::DscExtension; use crate::{dscresources::dscresource::DscResource, dscerror::DscError, progress::ProgressFormat}; use std::collections::BTreeMap; -use command_discovery::ManifestResource; +use command_discovery::ImportedManifest; use tracing::error; #[derive(Clone)] @@ -42,12 +42,12 @@ impl Discovery { /// # Returns /// /// A vector of `DscResource` instances. - pub fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str, progress_format: ProgressFormat) -> Vec { + pub fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str, progress_format: ProgressFormat) -> Vec { let discovery_types: Vec> = vec![ Box::new(command_discovery::CommandDiscovery::new(progress_format)), ]; - let mut resources: Vec = Vec::new(); + let mut resources: Vec = Vec::new(); for mut discovery_type in discovery_types { diff --git a/dsc_lib/src/extensions/dscextension.rs b/dsc_lib/src/extensions/dscextension.rs index b6e4c8f2..52e1c578 100644 --- a/dsc_lib/src/extensions/dscextension.rs +++ b/dsc_lib/src/extensions/dscextension.rs @@ -8,7 +8,7 @@ use schemars::JsonSchema; use std::{fmt::Display, path::Path}; use tracing::info; -use crate::{discovery::command_discovery::{load_manifest, ManifestResource}, dscerror::DscError, dscresources::{command_resource::{invoke_command, process_args}, dscresource::DscResource}}; +use crate::{discovery::command_discovery::{load_manifest, ImportedManifest}, dscerror::DscError, dscresources::{command_resource::{invoke_command, process_args}, dscresource::DscResource}}; use super::{discover::DiscoverResult, extension_manifest::ExtensionManifest}; @@ -110,7 +110,7 @@ impl DscExtension { } let manifest_path = Path::new(&discover_result.resource_manifest_path); // Currently we don't support extensions discovering other extensions - if let ManifestResource::Resource(resource) = load_manifest(manifest_path)? { + if let ImportedManifest::Resource(resource) = load_manifest(manifest_path)? { resources.push(resource); } } diff --git a/dsc_lib/src/lib.rs b/dsc_lib/src/lib.rs index e0ffc313..9c56026e 100644 --- a/dsc_lib/src/lib.rs +++ b/dsc_lib/src/lib.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::discovery::command_discovery::ManifestResource; +use crate::discovery::command_discovery::ImportedManifest; use crate::discovery::discovery_trait::DiscoveryKind; use crate::progress::ProgressFormat; @@ -51,7 +51,7 @@ impl DscManager { self.discovery.find_resource(name) } - pub fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str, progress_format: ProgressFormat) -> Vec { + pub fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str, progress_format: ProgressFormat) -> Vec { self.discovery.list_available(kind, type_name_filter, adapter_name_filter, progress_format) }