Bump to NatML 1.1.4

Author: lanreolokoba

Assembly-CSharp.csproj

@@ -1,6 +1,6 @@
 /* 
 *   NanoDet
-*   Copyright (c) 2022 NatML Inc. All Rights Reserved.
+*   Copyright © 2023 NatML Inc. All Rights Reserved.
 */
 
 namespace NatML.Examples.Visualizers {

Assets/NanoDetDetection.cs

@@ -23,27 +23,24 @@ public sealed class NanoDetSample : MonoBehaviour {
         private NanoDetPredictor predictor;
 
         private async void Start () {
-            // Fetch the model data from NatML Hub
-            modelData = await MLModelData.FromHub("@natsuite/nanodet");
-            // Create the model
-            model = new MLEdgeModel(modelData);
             // Create the NanoDet predictor
-            predictor = new NanoDetPredictor(model, modelData.labels);
+            predictor = await NanoDetPredictor.Create();
             // Listen for camera frames
-            cameraManager.OnFrame.AddListener(OnCameraFrame);
+            cameraManager.OnCameraFrame.AddListener(OnCameraFrame);
         }
 
         private void OnCameraFrame (CameraFrame frame) {
-            // Create input feature
-            var feature = frame.feature;
-            (feature.mean, feature.std) = modelData.normalization;
-            feature.aspectMode = modelData.aspectMode;
             // Detect
-            var detections = predictor.Predict(feature);
+            var detections = predictor.Predict(frame);
             // Visualize
             visualizer.Render(detections);
         }
 
-        void OnDisable () => model?.Dispose(); // Dispose the model
+        void OnDisable () {
+            // Stop listening for camera frames
+            cameraManager.OnCameraFrame.RemoveListener(OnCameraFrame);
+            // Dispose the model
+            model?.Dispose();
+        }
     }
 }

Assets/NanoDetSample.cs

@@ -38,7 +38,7 @@ RenderSettings:
   m_ReflectionIntensity: 1
   m_CustomReflection: {fileID: 0}
   m_Sun: {fileID: 705507994}
-  m_IndirectSpecularColor: {r: 0.44657838, g: 0.49641234, b: 0.57481676, a: 1}
+  m_IndirectSpecularColor: {r: 0.4440765, g: 0.49331635, b: 0.57239074, a: 1}
   m_UseRadianceAmbientProbe: 0
 --- !u!157 &3
 LightmapSettings:
@@ -588,6 +588,7 @@ GameObject:
   - component: {fileID: 1627475241}
   - component: {fileID: 1627475243}
   - component: {fileID: 1627475242}
+  - component: {fileID: 1627475244}
   m_Layer: 5
   m_Name: Preview
   m_TagString: Untagged
@@ -691,6 +692,25 @@ MonoBehaviour:
   m_EditorClassIdentifier: 
   cameraManager: {fileID: 1924049083}
   viewMode: 0
+  focusMode: 0
+  exposureMode: 0
+  zoomMode: 0
+  OnPresent:
+    m_PersistentCalls:
+      m_Calls: []
+--- !u!114 &1627475244
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1627475237}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: d0b148fe25e99eb48b9724523833bab1, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Delegates: []
 --- !u!1 &1924049080
 GameObject:
   m_ObjectHideFlags: 0
@@ -750,12 +770,14 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 98622613d505143a9a06b4b11b0a82cf, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  resolution: 3
-  capabilities: 1
+  capabilities: 2
   playOnAwake: 1
+  _facing: 0
+  resolution: 3
+  frameRate: 30
   focusMode: 0
   exposureMode: 0
-  OnFrame:
+  OnCameraFrame:
     m_PersistentCalls:
       m_Calls: []
 --- !u!1 &1934276594
@@ -788,6 +810,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 4f231c4fb786f3946a6b90b886c48677, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  m_SendPointerHoverToParent: 1
   m_HorizontalAxis: Horizontal
   m_VerticalAxis: Vertical
   m_SubmitButton: Submit

Assets/NanoDetSample.unity

@@ -8,6 +8,7 @@ namespace NatML.Examples.Visualizers {
     using System.Collections.Generic;
     using UnityEngine;
     using UnityEngine.UI;
+    using NatML.Vision;
     using VideoKit.UI;
 
     /// <summary>
@@ -21,29 +22,18 @@ public sealed class NanoDetVisualizer : MonoBehaviour {
 
 
         #region --Client API--
-        /// <summary>
-        /// Detection source image.
-        /// </summary>
-        public Texture2D image {
-            get => rawImage.texture as Texture2D;
-            set {
-                rawImage.texture = value;
-                aspectFitter.aspectRatio = (float)value.width / value.height;
-            }
-        }
-
         /// <summary>
         /// Render a set of object detections.
         /// </summary>
         /// <param name="image">Image which detections are made on.</param>
         /// <param name="detections">Detections to render.</param>
-        public void Render (params (Rect rect, string label, float score)[] detections) {
+        public void Render (params NanoDetPredictor.Detection[] detections) {
             // Delete current
             foreach (var rect in currentRects)
                 GameObject.Destroy(rect.gameObject);
             currentRects.Clear();
             // Render rects
-            var imageRect = new Rect(0, 0, image.width, image.height);
+            var imageRect = new Rect(0, 0, rawImage.texture.width, rawImage.texture.height);
             foreach (var detection in detections) {
                 var rect = Instantiate(detectionPrefab, transform);
                 rect.gameObject.SetActive(true);
@@ -56,13 +46,9 @@ public void Render (params (Rect rect, string label, float score)[] detections)
 
         #region --Operations--
         private RawImage rawImage;
-        private AspectRatioFitter aspectFitter;
         private readonly List<NanoDetDetection> currentRects = new List<NanoDetDetection>();
 
-        void Awake () {
-            rawImage = GetComponent<RawImage>();
-            aspectFitter = GetComponent<AspectRatioFitter>();
-        }
+        void Awake () => rawImage = GetComponent<RawImage>();
         #endregion
     }
 }

Assets/NanoDetVisualizer.cs

@@ -1,2 +1,8 @@
+## 1.0.1
++ Added `NanoDetPredictor.Create` static method for creating the predictor.
++ Added `NanoDetPredictor.Tag` constant string for model embedding.
++ Removed `NanoDetPredictor` public constructor. Use the `Create` static method instead.
++ Upgraded to NatML 1.1.4.
+
 ## 1.0.0
 + First release.

Packages/ai.natml.vision.nanodet/Changelog.md

@@ -13,7 +13,7 @@ Add the following items to your Unity project's `Packages/manifest.json`:
     }
   ],
   "dependencies": {
-    "ai.natml.vision.nanodet": "1.0.0"
+    "ai.natml.vision.nanodet": "1.0.1"
   }
 }
 ```
@@ -22,36 +22,26 @@ Add the following items to your Unity project's `Packages/manifest.json`:
 ## Detecting Objects in an Image
 First, create the NanoDet predictor:
 ```csharp
-// Fetch the model data from NatML Hub
-var modelData = await MLModelData.FromHub("@natsuite/nanodet");
-// Deserialize the model
-var model = modelData.Deserialize();
 // Create the NanoDet predictor
-var predictor = new NanoDetPredictor(model, modelData.labels);
+var predictor = await NanoDetPredictor.Create();
 ```
 
 Then detect objects in the image:
 ```csharp
 // Create image feature
 Texture2D image = ...;
-var imageFeature = new MLImageFeature(image);
-(imageFeature.mean, imageFeature.std) = modelData.normalization;
-imageFeature.aspectMode = modelData.aspectMode;
 // Detect objects
-(Rect rect, string label, float score)[] detections = predictor.Predict(imageFeature);
+NanoDetPredictor.Detection[] detections = predictor.Predict(image);
 ```
-
-> The detection rects are provided in normalized coordinates in range `[0.0, 1.0]`. The score is also normalized in range `[0.0, 1.0]`.
 ___
 
 ## Requirements
 - Unity 2021.2+
 
 ## Quick Tips
-- Join the [NatML community on Discord](https://hub.natml.ai/community).
+- Join the [NatML community on Discord](https://natml.ai/community).
 - Discover more ML models on [NatML Hub](https://hub.natml.ai).
 - See the [NatML documentation](https://docs.natml.ai/unity).
-- Discuss [NatML on Unity Forums](https://forum.unity.com/threads/open-beta-natml-machine-learning-runtime.1109339/).
 - Contact us at [hi@natml.ai](mailto:hi@natml.ai).
 
 Thank you very much!

Packages/ai.natml.vision.nanodet/README.md

@@ -1,13 +1,14 @@
 /* 
 *   NanoDet
-*   Copyright (c) 2022 NatML Inc. All Rights Reserved.
+*   Copyright © 2023 NatML Inc. All Rights Reserved.
 */
 
 namespace NatML.Vision {
 
     using System;
     using System.Collections.Generic;
     using System.Runtime.CompilerServices;
+    using System.Threading.Tasks;
     using System.Linq;
     using UnityEngine;
     using NatML.Features;
@@ -19,41 +20,44 @@ namespace NatML.Vision {
     /// This predictor accepts an image feature and produces a list of detections.
     /// Each detection is comprised of a normalized rect, label, and detection score.
     /// </summary>
-    public sealed class NanoDetPredictor : IMLPredictor<(Rect rect, string label, float score)[]> {
+    public sealed class NanoDetPredictor : IMLPredictor<NanoDetPredictor.Detection[]> {
 
-        #region --Client API--
+        #region --Types--
         /// <summary>
-        /// Class labels.
+        /// Detection.
         /// </summary>
-        public readonly string[] labels;
+        public struct Detection {
+            
+            /// <summary>
+            /// Normalized detection rect.
+            /// </summary>
+            public Rect rect;
+            
+            /// <summary>
+            /// Detection label.
+            /// </summary>
+            public string label;
+
+            /// <summary>
+            /// Normalized detection score.
+            /// </summary>
+            public float score;
+        }
+        #endregion
+
 
+        #region --Client API--
         /// <summary>
-        /// Create the NanoDet predictor.
+        /// Predictor tag.
         /// </summary>
-        /// <param name="model">NanoDet ML model.</param>
-        /// <param name="labels">Classification labels.</param>
-        /// <param name="minScore">Minimum candidate score.</param>
-        /// <param name="maxIoU">Maximum intersection-over-union score for overlap removal.</param>
-        public NanoDetPredictor (MLModel model, string[] labels, float minScore = 0.35f, float maxIoU = 0.5f) {
-            this.model = model as MLEdgeModel;
-            this.labels = labels;
-            this.minScore = minScore;
-            this.maxIoU = maxIoU;
-            this.inputType = model.inputs[0] as MLImageType;
-            this.anchors8 = GenerateAnchors(inputType.width, inputType.height, 8);
-            this.anchors16 = GenerateAnchors(inputType.width, inputType.height, 16);
-            this.anchors32 = GenerateAnchors(inputType.width, inputType.height, 32);
-            this.candidateBoxes = new List<Rect>();
-            this.candidateScores = new List<float>();
-            this.candidateLabels = new List<string>();
-        }
+        public const string Tag = "@natsuite/nanodet";
 
         /// <summary>
         /// Detect objects in an image.
         /// </summary>
         /// <param name="inputs">Input image.</param>
         /// <returns>Detected objects.</returns>
-        public unsafe (Rect rect, string label, float score)[] Predict (params MLFeature[] inputs) {
+        public unsafe Detection[] Predict (params MLFeature[] inputs) {
             // Check
             if (inputs.Length != 1)
                 throw new ArgumentException(@"NanoDet predictor expects a single feature", nameof(inputs));
@@ -63,6 +67,11 @@ public unsafe (Rect rect, string label, float score)[] Predict (params MLFeature
             var imageFeature = input as MLImageFeature;
             if (!imageType)
                 throw new ArgumentException(@"NanoDet predictor expects an an array or image feature", nameof(inputs));
+            // Preprocess
+            if (imageFeature != null) {
+                (imageFeature.mean, imageFeature.std) = model.normalization;
+                imageFeature.aspectMode = model.aspectMode;
+            }
             // Predict
             using var inputFeature = (input as IMLEdgeFeature).Create(inputType);
             using var outputFeatures = model.Predict(inputFeature);
@@ -118,16 +127,45 @@ public unsafe (Rect rect, string label, float score)[] Predict (params MLFeature
                     // Add
                     candidateBoxes.Add(box);
                     candidateScores.Add(score);
-                    candidateLabels.Add(labels[label]);
+                    candidateLabels.Add(model.labels[label]);
                 }
             }
             var keepIdx = MLImageFeature.NonMaxSuppression(candidateBoxes, candidateScores, maxIoU);
-            var result = new List<(Rect, string, float)>();
-            foreach (var idx in keepIdx)
-                result.Add((candidateBoxes[idx], candidateLabels[idx], candidateScores[idx]));
+            var result = new List<Detection>();
+            foreach (var idx in keepIdx) {
+                var detection = new Detection {
+                    rect = candidateBoxes[idx],
+                    label = candidateLabels[idx],
+                    score = candidateScores[idx]
+                };
+                result.Add(detection);
+            }
             // Return
             return result.ToArray();
         }
+
+        /// <summary>
+        /// Dispose the model and release resources.
+        /// </summary>
+        public void Dispose () => model.Dispose();
+
+        /// <summary>
+        /// Create the NanoDet predictor.
+        /// </summary>
+        /// <param name="minScore">Minimum candidate score.</param>
+        /// <param name="maxIoU">Maximum intersection-over-union score for overlap removal.</param>
+        /// <param name="configuration">Edge model configuration.</param>
+        /// <param name="accessKey">NatML access key.</param>
+        public static async Task<NanoDetPredictor> Create (
+            float minScore = 0.35f,
+            float maxIoU = 0.5f,
+            MLEdgeModel.Configuration configuration = null,
+            string accessKey = null
+        ) {
+            var model = await MLEdgeModel.Create(Tag, configuration, accessKey);
+            var predictor = new NanoDetPredictor(model, minScore, maxIoU);
+            return predictor;
+        }
         #endregion
 
 
@@ -143,7 +181,18 @@ public unsafe (Rect rect, string label, float score)[] Predict (params MLFeature
         private readonly List<float> candidateScores;
         private readonly List<string> candidateLabels;
 
-        void IDisposable.Dispose () { } // Not used
+        private NanoDetPredictor (MLModel model, float minScore, float maxIoU) {
+            this.model = model as MLEdgeModel;
+            this.minScore = minScore;
+            this.maxIoU = maxIoU;
+            this.inputType = model.inputs[0] as MLImageType;
+            this.anchors8 = GenerateAnchors(inputType.width, inputType.height, 8);
+            this.anchors16 = GenerateAnchors(inputType.width, inputType.height, 16);
+            this.anchors32 = GenerateAnchors(inputType.width, inputType.height, 32);
+            this.candidateBoxes = new List<Rect>();
+            this.candidateScores = new List<float>();
+            this.candidateLabels = new List<string>();
+        }
 
         private static Vector2[] GenerateAnchors (int width, int height, int stride) {
             var gridWidth = Mathf.FloorToInt((float)width / stride);