add execution of extension during discovery

Author: SteveL-MSFT

dsc/src/subcommand.rs

@@ -624,7 +624,7 @@ fn list_extensions(dsc: &mut DscManager, extension_name: Option<&String>, format
         write_table = true;
     }
     let mut include_separator = false;
-    for manifest_resource in dsc.list_available(&DiscoveryKind::Extension, extension_name.unwrap_or(&String::from("*")), &String::new(), progress_format) {
+    for manifest_resource in dsc.list_available(&DiscoveryKind::Extension, extension_name.unwrap_or(&String::from("*")), "", progress_format) {
         if let ManifestResource::Extension(extension) = manifest_resource {
             let mut capabilities = "-".to_string();
             let capability_types = [

dsc/tests/dsc_extension_discover.ps1

@@ -0,0 +1,24 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+Describe 'Discover extension tests' {
+    BeforeAll {
+        $oldPath = $env:PATH
+        $separator = [System.IO.Path]::PathSeparator
+        $env:PATH = "$PSScriptRoot$separator$oldPath"
+    }
+
+    AfterAll {
+        $env:PATH = $oldPath
+    }
+
+    It 'Discover extensions' {
+        $out = dsc extension list | ConvertFrom-Json
+        $LASTEXITCODE | Should -Be 0
+        $out.Count | Should -Be 1
+        $out.type | Should -BeExactly 'Test/Discover'
+        $out.version | Should -BeExactly '0.1.0'
+        $out.capabilities | Should -BeExactly @('discover')
+        $out.manifest | Should -Not -BeNullOrEmpty
+    }
+}

dsc_lib/locales/en-us.toml

@@ -81,12 +81,17 @@ usingResourcePath = "Using Resource Path: %{path}"
 discoverResources = "Discovering resources using filter: %{filter}"
 invalidAdapterFilter = "Could not build Regex filter for adapter name"
 progressSearching = "Searching for resources"
+extensionSearching = "Searching for extensions"
 foundResourceManifest = "Found resource manifest: %{path}"
 extensionFound = "Extension '%{extension}' found"
 adapterFound = "Resource adapter '%{adapter}' found"
 resourceFound = "Resource '%{resource}' found"
 executableNotFound = "Executable '%{executable}' not found for operation '%{operation}' for resource '%{resource}'"
 extensionInvalidVersion = "Extension '%{extension}' version '%{version}' is invalid"
+invalidManifest = "Invalid manifest for resource '%{resource}'"
+extensionResourceFound = "Extension found resource '%{resource}'"
+callingExtension = "Calling extension '%{extension}' to discover resources"
+extensionFoundResources = "Extension '%{extension}' found %{count} resources"
 
 [dscresources.commandResource]
 invokeGet = "Invoking get for '%{resource}'"
@@ -165,6 +170,9 @@ diffMissingItem = "diff: actual array missing expected item"
 resourceManifestSchemaTitle = "Resource manifest schema URI"
 resourceManifestSchemaDescription = "Defines the JSON Schema the resource manifest adheres to."
 
+[extensions.dscextension]
+discoverNoResults = "No results returned for discovery extension '%{extension}'"
+
 [extensions.extension_manifest]
 extensionManifestSchemaTitle = "Extension manifest schema URI"
 extensionManifestSchemaDescription = "Defines the JSON Schema the extension manifest adheres to."
@@ -339,6 +347,8 @@ unknown = "Unknown"
 unrecognizedSchemaUri = "Unrecognized $schema URI"
 validation = "Validation"
 validSchemaUrisAre = "Valid schema URIs are"
+extension = "Extension"
+unsupportedCapability = "does not support capability"
 setting = "Setting"
 
 [progress]

dsc_lib/src/discovery/command_discovery.rs

@@ -7,7 +7,7 @@ use crate::dscresources::dscresource::{Capability, DscResource, ImplementedAs};
 use crate::dscresources::resource_manifest::{import_manifest, validate_semver, Kind, ResourceManifest, SchemaKind};
 use crate::dscresources::command_resource::invoke_command;
 use crate::dscerror::DscError;
-use crate::extensions::dscextension::{self, DscExtension};
+use crate::extensions::dscextension::{self, DscExtension, Capability as ExtensionCapability};
 use crate::extensions::extension_manifest::ExtensionManifest;
 use crate::progress::{ProgressBar, ProgressFormat};
 use linked_hash_map::LinkedHashMap;
@@ -68,6 +68,7 @@ impl Default for ResourcePathSetting {
 }
 
 impl CommandDiscovery {
+    #[must_use]
     pub fn new(progress_format: ProgressFormat) -> CommandDiscovery {
         CommandDiscovery {
             adapters: BTreeMap::new(),
@@ -184,6 +185,11 @@ impl ResourceDiscovery for CommandDiscovery {
     fn discover(&mut self, kind: &DiscoveryKind, filter: &str) -> Result<(), DscError> {
         info!("{}", t!("discovery.commandDiscovery.discoverResources", filter = filter));
 
+        // if kind is DscResource, we need to discover extensions first
+        if *kind == DiscoveryKind::Resource {
+            self.discover(&DiscoveryKind::Extension, "*")?;
+        }
+
         let regex_str = convert_wildcard_to_regex(filter);
         debug!("Using regex {regex_str} as filter for adapter name");
         let mut regex_builder = RegexBuilder::new(&regex_str);
@@ -193,7 +199,14 @@ impl ResourceDiscovery for CommandDiscovery {
         };
 
         let mut progress = ProgressBar::new(1, self.progress_format)?;
-        progress.write_activity(t!("discovery.commandDiscovery.progressSearching").to_string().as_str());
+        match kind {
+            DiscoveryKind::Resource => {
+                progress.write_activity(t!("discovery.commandDiscovery.progressSearching").to_string().as_str());
+            },
+            DiscoveryKind::Extension => {
+                progress.write_activity(t!("discovery.commandDiscovery.extensionSearching").to_string().as_str());
+            }
+        }
 
         let mut adapters = BTreeMap::<String, Vec<DscResource>>::new();
         let mut resources = BTreeMap::<String, Vec<DscResource>>::new();
@@ -217,7 +230,7 @@ impl ResourceDiscovery for CommandDiscovery {
                             let file_name_lowercase = file_name.to_lowercase();
                             if (kind == &DiscoveryKind::Resource && DSC_RESOURCE_EXTENSIONS.iter().any(|ext| file_name_lowercase.ends_with(ext))) ||
                                 (kind == &DiscoveryKind::Extension && DSC_EXTENSION_EXTENSIONS.iter().any(|ext| file_name_lowercase.ends_with(ext))) {
-                                trace!("{}", t!("discovery.commandDiscovery.foundManifest", path = path.to_string_lossy()));
+                                trace!("{}", t!("discovery.commandDiscovery.foundResourceManifest", path = path.to_string_lossy()));
                                 let resource = match load_manifest(&path)
                                 {
                                     Ok(r) => r,
@@ -274,10 +287,31 @@ impl ResourceDiscovery for CommandDiscovery {
         }
 
         progress.write_increment(1);
-        debug!("Found {} matching non-adapter-based resources", resources.len());
-        self.adapters = adapters;
-        self.resources = resources;
-        self.extensions = extensions;
+
+        match kind {
+            DiscoveryKind::Resource => {
+                // Now we need to call discover extensions and add those resource to the list of resources
+                for (_extension_name, extension) in self.extensions.iter() {
+                    if extension.capabilities.contains(&ExtensionCapability::Discover) {
+                        debug!("{}", t!("discovery.commandDiscovery.callingExtension", extension = extension.type_name));
+                        let discovered_resources = extension.discover()?;
+                        debug!("{}", t!("discovery.commandDiscovery.extensionFoundResources", extension = extension.type_name, count = discovered_resources.len()));
+                        for resource in discovered_resources {
+                            if regex.is_match(&resource.type_name) {
+                                trace!("{}", t!("discovery.commandDiscovery.extensionResourceFound", resource = resource.type_name));
+                                insert_resource(&mut resources, &resource, true);
+                            }
+                        }
+                    }
+                }
+                self.adapters = adapters;
+                self.resources = resources;
+            },
+            DiscoveryKind::Extension => {
+                self.extensions = extensions;
+            }
+        }
+
         Ok(())
     }
 
@@ -505,7 +539,6 @@ impl ResourceDiscovery for CommandDiscovery {
     }
 }
 
-// helper to insert a resource into a vector of resources in order of newest to oldest
 // TODO: This should be a BTreeMap of the resource name and a BTreeMap of the version and DscResource, this keeps it version sorted more efficiently
 fn insert_resource(resources: &mut BTreeMap<String, Vec<DscResource>>, resource: &DscResource, skip_duplicate_version: bool) {
     if resources.contains_key(&resource.type_name) {
@@ -548,36 +581,48 @@ fn insert_resource(resources: &mut BTreeMap<String, Vec<DscResource>>, resource:
     }
 }
 
-fn load_manifest(path: &Path) -> Result<ManifestResource, DscError> {
+/// Loads a manifest from the given path and returns a `ManifestResource`.
+///
+/// # Arguments
+///
+/// * `path` - The path to the manifest file.
+///
+/// # Returns
+///
+/// * `ManifestResource` if the manifest was loaded successfully.
+///
+/// # Errors
+///
+/// * Returns a `DscError` if the manifest could not be loaded or parsed.
+pub fn load_manifest(path: &Path) -> Result<ManifestResource, DscError> {
     let contents = fs::read_to_string(path)?;
     if path.extension() == Some(OsStr::new("json")) {
-        if let Ok(manifest) = serde_json::from_str::<ResourceManifest>(&contents) {
-            let resource = load_resource_manifest(path, &manifest)?;
-            return Ok(ManifestResource::Resource(resource));
+        if let Ok(manifest) = serde_json::from_str::<ExtensionManifest>(&contents) {
+            let extension = load_extension_manifest(path, &manifest)?;
+            return Ok(ManifestResource::Extension(extension));
         }
-        let manifest = match serde_json::from_str::<ExtensionManifest>(&contents) {
+        let manifest = match serde_json::from_str::<ResourceManifest>(&contents) {
             Ok(manifest) => manifest,
             Err(err) => {
-                return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}")));
+                return Err(DscError::Manifest(t!("discovery.commandDiscovery.invalidManifest", resource = path.to_string_lossy()).to_string(), err));
             }
         };
-        let extension = load_extension_manifest(path, &manifest)?;
-        return Ok(ManifestResource::Extension(extension));
+        let resource = load_resource_manifest(path, &manifest)?;
+        return Ok(ManifestResource::Resource(resource));
     }
-    else {
-        if let Ok(manifest) = serde_yaml::from_str::<ResourceManifest>(&contents) {
-            let resource = load_resource_manifest(path, &manifest)?;
-            return Ok(ManifestResource::Resource(resource));
-        }
-        let manifest = match serde_yaml::from_str::<ExtensionManifest>(&contents) {
-            Ok(manifest) => manifest,
-            Err(err) => {
-                return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}")));
-            }
-        };
-        let extension = load_extension_manifest(path, &manifest)?;
-        return Ok(ManifestResource::Extension(extension));
+
+    if let Ok(manifest) = serde_yaml::from_str::<ResourceManifest>(&contents) {
+        let resource = load_resource_manifest(path, &manifest)?;
+        return Ok(ManifestResource::Resource(resource));
     }
+    let manifest = match serde_yaml::from_str::<ExtensionManifest>(&contents) {
+        Ok(manifest) => manifest,
+        Err(err) => {
+            return Err(DscError::Validation(format!("Invalid manifest {path:?} version value: {err}")));
+        }
+    };
+    let extension = load_extension_manifest(path, &manifest)?;
+    Ok(ManifestResource::Extension(extension))
 }
 
 fn load_resource_manifest(path: &Path, manifest: &ResourceManifest) -> Result<DscResource, DscError> {
@@ -663,7 +708,7 @@ fn load_extension_manifest(path: &Path, manifest: &ExtensionManifest) -> Result<
         capabilities,
         path: path.to_str().unwrap().to_string(),
         directory: path.parent().unwrap().to_str().unwrap().to_string(),
-        manifest: Some(serde_json::to_value(manifest)?),
+        manifest: serde_json::to_value(manifest)?,
         ..Default::default()
     };
 

dsc_lib/src/discovery/discovery_trait.rs

@@ -13,8 +13,68 @@ pub enum DiscoveryKind {
 }
 
 pub trait ResourceDiscovery {
+    /// Discovery method to find resources.
+    ///
+    /// # Arguments
+    ///
+    /// * `kind` - The kind of discovery (e.g., Resource).
+    /// * `filter` - The filter for the resource type name.
+    ///
+    /// # Returns
+    ///
+    /// A result indicating success or failure.
+    ///
+    /// # Errors
+    ///
+    /// This function will return an error if the underlying discovery fails.
     fn discover(&mut self, kind: &DiscoveryKind, filter: &str) -> Result<(), DscError>;
+
+    /// Discover adapted resources based on the provided filters.
+    ///
+    /// # Arguments
+    ///
+    /// * `name_filter` - The filter for the resource name.
+    /// * `adapter_filter` - The filter for the adapter name.
+    ///
+    /// # Returns
+    ///
+    /// A result indicating success or failure.
+    ///
+    /// # Errors
+    ///
+    /// This function will return an error if the underlying discovery fails.
     fn discover_adapted_resources(&mut self, name_filter: &str, adapter_filter: &str) -> Result<(), DscError>;
+
+    /// List available resources based on the provided filters.
+    ///
+    /// # Arguments
+    ///
+    /// * `kind` - The kind of discovery (e.g., Resource).
+    /// * `type_name_filter` - The filter for the resource type name.
+    /// * `adapter_name_filter` - The filter for the adapter name (only applies to resources).
+    ///
+    /// # Returns
+    ///
+    /// A result containing a map of resource names to their corresponding `ManifestResource` instances.
+    ///
+    /// # Errors
+    ///
+    /// This function will return an error if the underlying discovery fails.
     fn list_available(&mut self, kind: &DiscoveryKind, type_name_filter: &str, adapter_name_filter: &str) -> Result<BTreeMap<String, Vec<ManifestResource>>, DscError>;
+
+    /// Find resources based on the required resource types.
+    /// This is not applicable for extensions.
+    ///
+    /// # Arguments
+    ///
+    /// * `required_resource_types` - A slice of strings representing the required resource types.
+    ///
+    /// # Returns
+    ///
+    /// A result containing a map of resource names to their corresponding `DscResource` instances.
+    ///
+    /// # Errors
+    ///
+    /// This function will return an error if the underlying discovery fails.
     fn find_resources(&mut self, required_resource_types: &[String]) -> Result<BTreeMap<String, DscResource>, DscError>;
 }

dsc_lib/src/discovery/mod.rs

@@ -9,7 +9,7 @@ use crate::extensions::dscextension::DscExtension;
 use crate::{dscresources::dscresource::DscResource, dscerror::DscError, progress::ProgressFormat};
 use std::collections::BTreeMap;
 use command_discovery::ManifestResource;
-use tracing::{debug, error};
+use tracing::error;
 
 #[derive(Clone)]
 pub struct Discovery {

dsc_lib/src/dscerror.rs

@@ -116,6 +116,9 @@ pub enum DscError {
     #[error("{t}: {0}.  {t2}: {1:?}", t = t!("dscerror.unrecognizedSchemaUri"), t2 = t!("dscerror.validSchemaUrisAre"))]
     UnrecognizedSchemaUri(String, Vec<String>),
 
+    #[error("{t} '{0}' {t2} '{1}'", t = t!("dscerror.extension"), t2 = t!("dscerror.unsupportedCapability"))]
+    UnsupportedCapability(String, String),
+
     #[error("{t}: {0}", t = t!("dscerror.validation"))]
     Validation(String),
 

dsc_lib/src/dscresources/command_resource.rs

@@ -693,7 +693,17 @@ pub fn invoke_command(executable: &str, args: Option<Vec<String>>, input: Option
         .block_on(run_process_async(executable, args, input, cwd, env, exit_codes))
 }
 
-fn process_args(args: Option<&Vec<ArgKind>>, value: &str) -> Option<Vec<String>> {
+/// Process the arguments for a command resource.
+///
+/// # Arguments
+///
+/// * `args` - The arguments to process
+/// * `value` - The value to use for JSON input arguments
+///
+/// # Returns
+///
+/// A vector of strings representing the processed arguments
+pub fn process_args(args: Option<&Vec<ArgKind>>, value: &str) -> Option<Vec<String>> {
     let Some(arg_values) = args else {
         debug!("{}", t!("dscresources.commandResource.noArgs"));
         return None;