From 473544776665e3f8eef08cd3d87c82715976d8c8 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Sun, 13 Nov 2022 00:45:15 +0800 Subject: [PATCH 01/18] Add `DataViewSchema` overloads --- .../OnnxExportExtensions.cs | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index df377e46d0..082aa0232a 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -79,7 +79,7 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, inputData).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema)).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -91,7 +91,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, inputData, opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -103,6 +103,41 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, inputData, outputColumns).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), outputColumns).WriteTo(stream); + + /// + /// Convert the specified to ONNX format and writes to a stream. + /// + /// The class that attached to. + /// The that will be converted into ONNX format. + /// The schema of the input to the transformer. + /// The stream to write the protobuf model to. + /// An ONNX model equivalent to the converted ML.NET model. + public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream) => + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema)).WriteTo(stream); + + /// + /// Convert the specified to ONNX format and writes to a stream. + /// + /// The class that attached to. + /// The that will be converted into ONNX format. + /// The schema of the input to the transformer. + /// The OpSet version to use for exporting the model. This value must be greater than or equal to 9 and less than or equal to 12 + /// The stream to write the protobuf model to. + /// An ONNX model equivalent to the converted ML.NET model. + public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, int opSetVersion, Stream stream) => + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema), opSetVersion).WriteTo(stream); + + /// + /// Convert the specified to ONNX format and writes to a stream. + /// + /// The class that attached to. + /// The that will be converted into ONNX format. + /// The schema of the input to the transformer. + /// The stream to write the protobuf model to. + /// List of output columns we want to keep. + /// An ONNX model equivalent to the converted ML.NET model. + public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream, params string[] outputColumns) => + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema), outputColumns).WriteTo(stream); } } From 8ca34dde83487c3d3d61649d2b71a15eaa597ad6 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Sun, 13 Nov 2022 01:00:10 +0800 Subject: [PATCH 02/18] Delete extra whitespaces --- src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 082aa0232a..66da88283d 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -104,7 +104,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream, params string[] outputColumns) => ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), outputColumns).WriteTo(stream); - + /// /// Convert the specified to ONNX format and writes to a stream. /// From 3e8349b905a6791397ca07d8b7de0ad4665553c3 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Sun, 13 Nov 2022 02:25:04 +0800 Subject: [PATCH 03/18] Try custom EmptyDataView --- .../OnnxExportExtensions.cs | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 66da88283d..555a909078 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -79,7 +79,7 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema)).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputData.Schema)).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -91,7 +91,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputData.Schema), opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -103,7 +103,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), outputColumns).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputData.Schema), outputColumns).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -114,7 +114,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema)).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputSchema)).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -126,7 +126,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema), opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputSchema), opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -138,6 +138,45 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema), outputColumns).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputSchema), outputColumns).WriteTo(stream); + } + + sealed record EmptyDataView(DataViewSchema Schema) : IDataView + { + public bool CanShuffle => true; + + public long? GetRowCount() => 0L; + + public DataViewRowCursor GetRowCursor(IEnumerable columnsNeeded, Random rand = null) + => new EmptyDataViewRowCursor(Schema); + + public DataViewRowCursor[] GetRowCursorSet(IEnumerable columnsNeeded, int n, Random rand = null) + => Array.Empty(); + } + + sealed class EmptyDataViewRowCursor : DataViewRowCursor + { + private readonly DataViewSchema schema; + + public EmptyDataViewRowCursor(DataViewSchema Schema) + { + schema = Schema; + } + + public override DataViewSchema Schema => schema; + + public override long Position => -1L; + + public override bool IsColumnActive(DataViewSchema.Column column) => false; + + public override bool MoveNext() => false; + + public override long Batch => 0L; + + public override ValueGetter GetGetter(DataViewSchema.Column column) + => throw new InvalidOperationException(); + + public override ValueGetter GetIdGetter() + => throw new InvalidOperationException(); } } From f412ab01731f8a6162e35fdea6bf9f474aa1a482 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Sun, 13 Nov 2022 02:38:37 +0800 Subject: [PATCH 04/18] Don't use `record` --- .../OnnxExportExtensions.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 555a909078..683efbd303 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -141,8 +142,17 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputSchema), outputColumns).WriteTo(stream); } - sealed record EmptyDataView(DataViewSchema Schema) : IDataView + sealed class EmptyDataView : IDataView { + private readonly DataViewSchema _schema; + + public EmptyDataView(DataViewSchema schema) + { + _schema = schema; + } + + public DataViewSchema Schema => _schema; + public bool CanShuffle => true; public long? GetRowCount() => 0L; From daacf2ec7a16da6fde2b3a6fedaf31bc3e9b4df1 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Sun, 13 Nov 2022 02:39:09 +0800 Subject: [PATCH 05/18] Delete extra space --- src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 683efbd303..07450b9ef7 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -145,7 +145,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform sealed class EmptyDataView : IDataView { private readonly DataViewSchema _schema; - + public EmptyDataView(DataViewSchema schema) { _schema = schema; From ba3aaef7430478251395726feb846cbea0867b3b Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Sun, 13 Nov 2022 02:53:36 +0800 Subject: [PATCH 06/18] Fix errors --- .../OnnxExportExtensions.cs | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 07450b9ef7..806cb2e0c2 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -140,53 +140,53 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream, params string[] outputColumns) => ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputSchema), outputColumns).WriteTo(stream); - } - - sealed class EmptyDataView : IDataView - { - private readonly DataViewSchema _schema; - public EmptyDataView(DataViewSchema schema) + private sealed class EmptyDataView : IDataView { - _schema = schema; - } + private readonly DataViewSchema _schema; - public DataViewSchema Schema => _schema; + public EmptyDataView(DataViewSchema schema) + { + _schema = schema; + } - public bool CanShuffle => true; + public DataViewSchema Schema => _schema; - public long? GetRowCount() => 0L; + public bool CanShuffle => true; - public DataViewRowCursor GetRowCursor(IEnumerable columnsNeeded, Random rand = null) - => new EmptyDataViewRowCursor(Schema); + public long? GetRowCount() => 0L; - public DataViewRowCursor[] GetRowCursorSet(IEnumerable columnsNeeded, int n, Random rand = null) - => Array.Empty(); - } + public DataViewRowCursor GetRowCursor(IEnumerable columnsNeeded, Random rand = null) + => new EmptyDataViewRowCursor(Schema); - sealed class EmptyDataViewRowCursor : DataViewRowCursor - { - private readonly DataViewSchema schema; + public DataViewRowCursor[] GetRowCursorSet(IEnumerable columnsNeeded, int n, Random rand = null) + => Array.Empty(); + } - public EmptyDataViewRowCursor(DataViewSchema Schema) + private sealed class EmptyDataViewRowCursor : DataViewRowCursor { - schema = Schema; - } + private readonly DataViewSchema _schema; - public override DataViewSchema Schema => schema; + public EmptyDataViewRowCursor(DataViewSchema schema) + { + _schema = schema; + } + + public override DataViewSchema Schema => _schema; - public override long Position => -1L; + public override long Position => -1L; - public override bool IsColumnActive(DataViewSchema.Column column) => false; + public override bool IsColumnActive(DataViewSchema.Column column) => false; - public override bool MoveNext() => false; + public override bool MoveNext() => false; - public override long Batch => 0L; + public override long Batch => 0L; - public override ValueGetter GetGetter(DataViewSchema.Column column) - => throw new InvalidOperationException(); + public override ValueGetter GetGetter(DataViewSchema.Column column) + => throw new InvalidOperationException(); - public override ValueGetter GetIdGetter() - => throw new InvalidOperationException(); + public override ValueGetter GetIdGetter() + => throw new InvalidOperationException(); + } } } From 7de47eeeb86eff84f44bbca84733a0bb718e483d Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Sun, 13 Nov 2022 05:03:23 +0800 Subject: [PATCH 07/18] Back to built-in `EmptyDataView` --- .../OnnxExportExtensions.cs | 61 ++----------------- 1 file changed, 6 insertions(+), 55 deletions(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 806cb2e0c2..66da88283d 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -80,7 +79,7 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputData.Schema)).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema)).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -92,7 +91,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputData.Schema), opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -104,7 +103,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputData.Schema), outputColumns).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), outputColumns).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -115,7 +114,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputSchema)).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema)).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -127,7 +126,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputSchema), opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema), opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -139,54 +138,6 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(inputSchema), outputColumns).WriteTo(stream); - - private sealed class EmptyDataView : IDataView - { - private readonly DataViewSchema _schema; - - public EmptyDataView(DataViewSchema schema) - { - _schema = schema; - } - - public DataViewSchema Schema => _schema; - - public bool CanShuffle => true; - - public long? GetRowCount() => 0L; - - public DataViewRowCursor GetRowCursor(IEnumerable columnsNeeded, Random rand = null) - => new EmptyDataViewRowCursor(Schema); - - public DataViewRowCursor[] GetRowCursorSet(IEnumerable columnsNeeded, int n, Random rand = null) - => Array.Empty(); - } - - private sealed class EmptyDataViewRowCursor : DataViewRowCursor - { - private readonly DataViewSchema _schema; - - public EmptyDataViewRowCursor(DataViewSchema schema) - { - _schema = schema; - } - - public override DataViewSchema Schema => _schema; - - public override long Position => -1L; - - public override bool IsColumnActive(DataViewSchema.Column column) => false; - - public override bool MoveNext() => false; - - public override long Batch => 0L; - - public override ValueGetter GetGetter(DataViewSchema.Column column) - => throw new InvalidOperationException(); - - public override ValueGetter GetIdGetter() - => throw new InvalidOperationException(); - } + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema), outputColumns).WriteTo(stream); } } From 0180ff645712568957cc72cdb5be013fb2737ccd Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Sun, 13 Nov 2022 13:15:07 +0800 Subject: [PATCH 08/18] Test build failure --- src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 66da88283d..d7f8239b5d 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -79,7 +79,7 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema)).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputData).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -91,7 +91,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputData, opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -103,7 +103,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), outputColumns).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputData, outputColumns).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. From 3c111b91942fef531e453a43afd81ef7d09f5824 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 01:05:10 +0800 Subject: [PATCH 09/18] Try again --- src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index d7f8239b5d..66da88283d 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -79,7 +79,7 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, inputData).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema)).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -91,7 +91,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, inputData, opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -103,7 +103,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, inputData, outputColumns).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), outputColumns).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. From 04d43e2e36903af867b3ffbcafd420773e528536 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 01:15:11 +0800 Subject: [PATCH 10/18] Change `ConvertToOnnxProtobuf` --- .../OnnxExportExtensions.cs | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 66da88283d..88207737fb 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -47,11 +47,35 @@ private static ModelProto ConvertToOnnxProtobufCore(IHostEnvironment env, OnnxCo /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. [BestFriend] - internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, string[] outputColumns = null) + internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, string[] outputColumns = null) => + ConvertToOnnxProtobuf(catalog, transform, inputData.Schema, outputColumns); + + /// + /// Convert the specified to ONNX format. Note that ONNX uses Google's Protobuf so the returned value is a Protobuf object. + /// + /// The class that attached to. + /// The that will be converted into ONNX format. + /// The input of the specified transform. + /// The OpSet version to use for exporting the model. This value must be greater than or equal to 9 and less than or equal to 12 + /// An ONNX model equivalent to the converted ML.NET model. + [BestFriend] + internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, int opSetVersion) => + ConvertToOnnxProtobuf(catalog, transform, inputData.Schema, opSetVersion); + + /// + /// Convert the specified to ONNX format. Note that ONNX uses Google's Protobuf so the returned value is a Protobuf object. + /// + /// The class that attached to. + /// The that will be converted into ONNX format. + /// The schema of the input to the transformer. + /// List of output columns we want to keep. + /// An ONNX model equivalent to the converted ML.NET model. + [BestFriend] + internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, string[] outputColumns = null) { var env = catalog.GetEnvironment(); var ctx = new OnnxContextImpl(env, "model", "ML.NET", "0", 0, "machinelearning.dotnet", OnnxVersion.Stable); - return ConvertToOnnxProtobufCore(env, ctx, transform, inputData, outputColumns); + return ConvertToOnnxProtobufCore(env, ctx, transform, new EmptyDataView(env, inputSchema), outputColumns); } /// @@ -59,15 +83,15 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// /// The class that attached to. /// The that will be converted into ONNX format. - /// The input of the specified transform. + /// The schema of the input to the transformer. /// The OpSet version to use for exporting the model. This value must be greater than or equal to 9 and less than or equal to 12 /// An ONNX model equivalent to the converted ML.NET model. [BestFriend] - internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, int opSetVersion) + internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, int opSetVersion) { var env = catalog.GetEnvironment(); var ctx = new OnnxContextImpl(env, "model", "ML.NET", "0", 0, "machinelearning.dotnet", OnnxVersion.Stable, opSetVersion); - return ConvertToOnnxProtobufCore(env, ctx, transform, inputData); + return ConvertToOnnxProtobufCore(env, ctx, transform, new EmptyDataView(env, inputSchema)); } /// @@ -79,7 +103,7 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema)).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputData).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -91,7 +115,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputData, opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -103,7 +127,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, IDataView inputData, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputData.Schema), outputColumns).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputData, outputColumns).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -114,7 +138,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema)).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputSchema).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -126,7 +150,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// The stream to write the protobuf model to. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, int opSetVersion, Stream stream) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema), opSetVersion).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputSchema, opSetVersion).WriteTo(stream); /// /// Convert the specified to ONNX format and writes to a stream. @@ -138,6 +162,6 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// List of output columns we want to keep. /// An ONNX model equivalent to the converted ML.NET model. public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransformer transform, DataViewSchema inputSchema, Stream stream, params string[] outputColumns) => - ConvertToOnnxProtobuf(catalog, transform, new EmptyDataView(catalog.GetEnvironment(), inputSchema), outputColumns).WriteTo(stream); + ConvertToOnnxProtobuf(catalog, transform, inputSchema, outputColumns).WriteTo(stream); } } From 070d159b14a4c096c361d8a20340bc432a9f5ed0 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 02:58:20 +0800 Subject: [PATCH 11/18] Add console log message for debug --- test/Microsoft.ML.TestFramework/BaseTestBaseline.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 71f9d2c4aa..945546f2e9 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -678,6 +678,9 @@ public void CompareResults(string leftColumnName, string rightColumnName, IDataV var leftColumn = left.Schema[leftColumnName]; var leftType = leftColumn.Type.GetItemType(); + Console.WriteLine("leftColumnName: " + leftColumnName); + Console.WriteLine("leftType: " + leftType); + if (leftType == NumberDataViewType.SByte) CompareSelectedColumns(leftColumnName, rightColumnName, left, right, isRightColumnOnnxScalar: isRightColumnOnnxScalar); else if (leftType == NumberDataViewType.Byte) @@ -709,6 +712,9 @@ private void CompareSelectedColumns(string leftColumnName, string rightColumn var leftColumn = left.Schema[leftColumnName]; var rightColumn = right.Schema[rightColumnName]; + Console.WriteLine("leftColumnName: " + leftColumnName); + Console.WriteLine("rightColumnName: " + rightColumnName); + using (var expectedCursor = left.GetRowCursor(leftColumn)) using (var actualCursor = right.GetRowCursor(rightColumn)) { @@ -743,7 +749,7 @@ private void CompareSelectedColumns(string leftColumnName, string rightColumn while (expectedCursor.MoveNext() && actualCursor.MoveNext()) { - + Console.WriteLine("MoveNext()"); if (leftColumn.Type is VectorDataViewType) { expectedVectorGetter(ref expectedVector); From 2317305a6470487e1d149f935759e6374846c6c3 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 11:59:50 +0800 Subject: [PATCH 12/18] Delete log in test framework --- test/Microsoft.ML.TestFramework/BaseTestBaseline.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 945546f2e9..71f9d2c4aa 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -678,9 +678,6 @@ public void CompareResults(string leftColumnName, string rightColumnName, IDataV var leftColumn = left.Schema[leftColumnName]; var leftType = leftColumn.Type.GetItemType(); - Console.WriteLine("leftColumnName: " + leftColumnName); - Console.WriteLine("leftType: " + leftType); - if (leftType == NumberDataViewType.SByte) CompareSelectedColumns(leftColumnName, rightColumnName, left, right, isRightColumnOnnxScalar: isRightColumnOnnxScalar); else if (leftType == NumberDataViewType.Byte) @@ -712,9 +709,6 @@ private void CompareSelectedColumns(string leftColumnName, string rightColumn var leftColumn = left.Schema[leftColumnName]; var rightColumn = right.Schema[rightColumnName]; - Console.WriteLine("leftColumnName: " + leftColumnName); - Console.WriteLine("rightColumnName: " + rightColumnName); - using (var expectedCursor = left.GetRowCursor(leftColumn)) using (var actualCursor = right.GetRowCursor(rightColumn)) { @@ -749,7 +743,7 @@ private void CompareSelectedColumns(string leftColumnName, string rightColumn while (expectedCursor.MoveNext() && actualCursor.MoveNext()) { - Console.WriteLine("MoveNext()"); + if (leftColumn.Type is VectorDataViewType) { expectedVectorGetter(ref expectedVector); From 77a4fea45b0bc694d3fc3cfd2cec233dd70301c7 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 12:05:10 +0800 Subject: [PATCH 13/18] Add log in EmptyDataView --- src/Microsoft.ML.Data/DataView/EmptyDataView.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/DataView/EmptyDataView.cs b/src/Microsoft.ML.Data/DataView/EmptyDataView.cs index 7432babf06..d2eff21177 100644 --- a/src/Microsoft.ML.Data/DataView/EmptyDataView.cs +++ b/src/Microsoft.ML.Data/DataView/EmptyDataView.cs @@ -47,7 +47,13 @@ private sealed class Cursor : RootCursorBase private readonly bool[] _active; public override DataViewSchema Schema { get; } - public override long Batch => 0; + public override long Batch + { + get { + Console.WriteLine("EmptyDataView.Cursor.Batch"); + return 0L; + } + } public Cursor(IChannelProvider provider, DataViewSchema schema, IEnumerable columnsNeeded) : base(provider) @@ -55,6 +61,7 @@ public Cursor(IChannelProvider provider, DataViewSchema schema, IEnumerable GetIdGetter() From a1a59fd878c7dc4467f4adace76f36c6b9df86c1 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 14:00:35 +0800 Subject: [PATCH 14/18] Fix formatting --- src/Microsoft.ML.Data/DataView/EmptyDataView.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/DataView/EmptyDataView.cs b/src/Microsoft.ML.Data/DataView/EmptyDataView.cs index d2eff21177..319c07b34d 100644 --- a/src/Microsoft.ML.Data/DataView/EmptyDataView.cs +++ b/src/Microsoft.ML.Data/DataView/EmptyDataView.cs @@ -49,7 +49,8 @@ private sealed class Cursor : RootCursorBase public override DataViewSchema Schema { get; } public override long Batch { - get { + get + { Console.WriteLine("EmptyDataView.Cursor.Batch"); return 0L; } From bf3ed067e2697f6ff9c6b3e51102586e227c0be8 Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 21:44:59 +0800 Subject: [PATCH 15/18] Try DataFrame --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 6fde298c52..6e41d2e259 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -9,6 +9,7 @@ using System.Text.RegularExpressions; using Google.Protobuf; using Microsoft.ML.Data; +using Microsoft.Data.Analysis; using Microsoft.ML.EntryPoints; using Microsoft.ML.Model.OnnxConverter; using Microsoft.ML.RunTests; @@ -1825,7 +1826,7 @@ public void UseKeyDataViewTypeAsUInt32InOnnxInput() { Assert.EndsWith(stdSuffix, name); var colName = name.Replace(stdSuffix, ""); - CompareResults(colName, colName, onnxResult, onnxResult2); + CompareResults(colName, colName, onnxResult.ToDataFrame(), onnxResult2.ToDataFrame()); } (onnxTransformer as IDisposable)?.Dispose(); (onnxTransformer2 as IDisposable)?.Dispose(); From db3a2b57328b2afa6670a6ccdbd4dae8e9be645e Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 21:51:40 +0800 Subject: [PATCH 16/18] Change log location --- src/Microsoft.ML.Data/DataView/EmptyDataView.cs | 10 +--------- src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs | 5 +++++ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.ML.Data/DataView/EmptyDataView.cs b/src/Microsoft.ML.Data/DataView/EmptyDataView.cs index 319c07b34d..7432babf06 100644 --- a/src/Microsoft.ML.Data/DataView/EmptyDataView.cs +++ b/src/Microsoft.ML.Data/DataView/EmptyDataView.cs @@ -47,14 +47,7 @@ private sealed class Cursor : RootCursorBase private readonly bool[] _active; public override DataViewSchema Schema { get; } - public override long Batch - { - get - { - Console.WriteLine("EmptyDataView.Cursor.Batch"); - return 0L; - } - } + public override long Batch => 0; public Cursor(IChannelProvider provider, DataViewSchema schema, IEnumerable columnsNeeded) : base(provider) @@ -62,7 +55,6 @@ public Cursor(IChannelProvider provider, DataViewSchema schema, IEnumerable GetIdGetter() diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs b/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs index 1f8ef34995..8f07e4afac 100644 --- a/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs +++ b/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs @@ -622,6 +622,11 @@ private void UpdateCacheIfNeeded(long position, INamedOnnxValueGetter[] srcNamed inputNameOnnxValues.Add(srcNamedOnnxValueGetters[i].GetNamedOnnxValue()); } + foreach (var v in inputNameOnnxValues) + { + Console.WriteLine(v.Name); + } + outputCache.OutputOnnxValues?.Dispose(); outputCache.OutputOnnxValues = _parent.Model.Run(inputNameOnnxValues, activeOutputColNames); Contracts.Assert(outputCache.OutputOnnxValues.Count > 0); From 33ef98276a2ecbf01dd662b6724024d091439afc Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 22:58:12 +0800 Subject: [PATCH 17/18] Clean up for review --- src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs | 5 ----- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 3 +-- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs b/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs index 8f07e4afac..1f8ef34995 100644 --- a/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs +++ b/src/Microsoft.ML.OnnxTransformer/OnnxTransform.cs @@ -622,11 +622,6 @@ private void UpdateCacheIfNeeded(long position, INamedOnnxValueGetter[] srcNamed inputNameOnnxValues.Add(srcNamedOnnxValueGetters[i].GetNamedOnnxValue()); } - foreach (var v in inputNameOnnxValues) - { - Console.WriteLine(v.Name); - } - outputCache.OutputOnnxValues?.Dispose(); outputCache.OutputOnnxValues = _parent.Model.Run(inputNameOnnxValues, activeOutputColNames); Contracts.Assert(outputCache.OutputOnnxValues.Count > 0); diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 6e41d2e259..6fde298c52 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -9,7 +9,6 @@ using System.Text.RegularExpressions; using Google.Protobuf; using Microsoft.ML.Data; -using Microsoft.Data.Analysis; using Microsoft.ML.EntryPoints; using Microsoft.ML.Model.OnnxConverter; using Microsoft.ML.RunTests; @@ -1826,7 +1825,7 @@ public void UseKeyDataViewTypeAsUInt32InOnnxInput() { Assert.EndsWith(stdSuffix, name); var colName = name.Replace(stdSuffix, ""); - CompareResults(colName, colName, onnxResult.ToDataFrame(), onnxResult2.ToDataFrame()); + CompareResults(colName, colName, onnxResult, onnxResult2); } (onnxTransformer as IDisposable)?.Dispose(); (onnxTransformer2 as IDisposable)?.Dispose(); From aed491f5cdce678e30b68ab8b949051b0934da7c Mon Sep 17 00:00:00 2001 From: Franklin Tse Date: Mon, 14 Nov 2022 23:01:40 +0800 Subject: [PATCH 18/18] Correct `catelog` param doc --- src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs index 88207737fb..88194c35b8 100644 --- a/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs +++ b/src/Microsoft.ML.OnnxConverter/OnnxExportExtensions.cs @@ -65,7 +65,7 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// /// Convert the specified to ONNX format. Note that ONNX uses Google's Protobuf so the returned value is a Protobuf object. /// - /// The class that attached to. + /// The class that attached to. /// The that will be converted into ONNX format. /// The schema of the input to the transformer. /// List of output columns we want to keep. @@ -81,7 +81,7 @@ internal static ModelProto ConvertToOnnxProtobuf(this ModelOperationsCatalog cat /// /// Convert the specified to ONNX format. Note that ONNX uses Google's Protobuf so the returned value is a Protobuf object. /// - /// The class that attached to. + /// The class that attached to. /// The that will be converted into ONNX format. /// The schema of the input to the transformer. /// The OpSet version to use for exporting the model. This value must be greater than or equal to 9 and less than or equal to 12 @@ -132,7 +132,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// /// Convert the specified to ONNX format and writes to a stream. /// - /// The class that attached to. + /// The class that attached to. /// The that will be converted into ONNX format. /// The schema of the input to the transformer. /// The stream to write the protobuf model to. @@ -143,7 +143,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// /// Convert the specified to ONNX format and writes to a stream. /// - /// The class that attached to. + /// The class that attached to. /// The that will be converted into ONNX format. /// The schema of the input to the transformer. /// The OpSet version to use for exporting the model. This value must be greater than or equal to 9 and less than or equal to 12 @@ -155,7 +155,7 @@ public static void ConvertToOnnx(this ModelOperationsCatalog catalog, ITransform /// /// Convert the specified to ONNX format and writes to a stream. /// - /// The class that attached to. + /// The class that attached to. /// The that will be converted into ONNX format. /// The schema of the input to the transformer. /// The stream to write the protobuf model to.