Skip to content

Commit a20d6e0

Browse files
JakeRadMSFTJakeRadMSFT
committed
Merge remote-tracking branch 'origin/main' into jakerad_generic_math
2 parents 6be198d + 8858ab6 commit a20d6e0

22 files changed

+353
-66
lines changed

‎.github/fabricbot.json

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@
406406
"subCapability": "IssueCommentResponder",
407407
"version": "1.0",
408408
"config": {
409-
"taskName": "Replace `needs-author-action` label with `needs-further-triage` label when the author comments on an issue",
409+
"taskName": "Replace `needs-author-action` label with `needs-further-triage` label when the author comments on an issue that is not still untriaged",
410410
"actions": [
411411
{
412412
"name": "addLabel",
@@ -448,6 +448,74 @@
448448
"label": "needs-author-action"
449449
}
450450
},
451+
{
452+
"operator": "not",
453+
"operands": [
454+
{
455+
"name": "hasLabel",
456+
"parameters": {
457+
"label": "untriaged"
458+
}
459+
}
460+
]
461+
},
462+
{
463+
"name": "isOpen",
464+
"parameters": {}
465+
}
466+
]
467+
}
468+
}
469+
},
470+
{
471+
"taskSource": "fabricbot-config",
472+
"taskType": "trigger",
473+
"capabilityId": "IssueResponder",
474+
"subCapability": "IssueCommentResponder",
475+
"version": "1.0",
476+
"config": {
477+
"taskName": "Remove `needs-author-action` label when the author comments on an `untriaged` issue",
478+
"actions": [
479+
{
480+
"name": "removeLabel",
481+
"parameters": {
482+
"label": "needs-author-action"
483+
}
484+
}
485+
],
486+
"eventType": "issue",
487+
"eventNames": [
488+
"issue_comment"
489+
],
490+
"conditions": {
491+
"operator": "and",
492+
"operands": [
493+
{
494+
"name": "isAction",
495+
"parameters": {
496+
"action": "created"
497+
}
498+
},
499+
{
500+
"name": "isActivitySender",
501+
"parameters": {
502+
"user": {
503+
"type": "author"
504+
}
505+
}
506+
},
507+
{
508+
"name": "hasLabel",
509+
"parameters": {
510+
"label": "needs-author-action"
511+
}
512+
},
513+
{
514+
"name": "hasLabel",
515+
"parameters": {
516+
"label": "untriaged"
517+
}
518+
},
451519
{
452520
"name": "isOpen",
453521
"parameters": {}

‎build/vsts-ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ stages:
100100
pool:
101101
vmImage: macOS-12
102102
steps:
103-
- script: brew update && rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
103+
- script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && brew update && rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
104104
displayName: Install build dependencies
105105
# Only build native assets to avoid conflicts.
106106
- script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true

‎eng/helix.proj

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,6 @@
9999
<HelixPreCommands Condition="$(IsPosixShell)">$(HelixPreCommands);export ML_TEST_DATADIR=$HELIX_CORRELATION_PAYLOAD;export MICROSOFTML_RESOURCE_PATH=$HELIX_WORKITEM_ROOT;sudo chmod -R 777 $HELIX_WORKITEM_ROOT;sudo chown -R $USER $HELIX_WORKITEM_ROOT</HelixPreCommands>
100100
<HelixPreCommands Condition="!$(IsPosixShell)">$(HelixPreCommands);set ML_TEST_DATADIR=%HELIX_CORRELATION_PAYLOAD%;set MICROSOFTML_RESOURCE_PATH=%HELIX_WORKITEM_ROOT%</HelixPreCommands>
101101

102-
<HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx'))">$(HelixPreCommands);install_name_tool -change "/usr/local/opt/libomp/lib/libomp.dylib" "@loader_path/libomp.dylib" libSymSgdNative.dylib</HelixPreCommands>
103-
<HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx'))">$(HelixPreCommands);install_name_tool -change "/usr/local/opt/libomp/lib/libomp.dylib" "@loader_path/libomp.dylib" runtimes/osx-x64/native/lib_lightgbm.dylib</HelixPreCommands>
104-
<HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx'))">$(HelixPreCommands);install_name_tool -change "/usr/local/opt/libomp/lib/libomp.dylib" "@loader_path/libomp.dylib" runtimes/osx-x64/native/libonnxruntime.dylib</HelixPreCommands>
105102
<HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx'))">$(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;</HelixPreCommands>
106103

107104
<HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('armarch'))">$(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y</HelixPreCommands>
@@ -122,6 +119,15 @@
122119
DestinationFolder="$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)">
123120
</Copy>
124121

122+
<Exec Condition="Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\libSymSgdNative.dylib') AND $(HelixTargetQueues.ToLowerInvariant().Contains('osx'))"
123+
Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@loader_path/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/libSymSgdNative.dylib" />
124+
125+
<Exec Condition="Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\runtimes\osx-x64\native\lib_lightgbm.dylib') AND $(HelixTargetQueues.ToLowerInvariant().Contains('osx'))"
126+
Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@loader_path/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/runtimes/osx-x64/native/lib_lightgbm.dylib" />
127+
128+
<Exec Condition="Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\runtimes\osx-x64\native\libonnxruntime.dylib') AND $(HelixTargetQueues.ToLowerInvariant().Contains('osx'))"
129+
Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@loader_path/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/runtimes/osx-x64/native/libonnxruntime.dylib" />
130+
125131
<!-- Remove the native libraries for other OS to save on payload size -->
126132
<ItemGroup>
127133
<WindowsFiles Include="$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\runtimes\win*\**\*.*"/>

‎src/Microsoft.Data.Analysis/DataFrameColumnCollection.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Licensed to the .NET Foundation under one or more agreements.
1+
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

‎src/Microsoft.Data.Analysis/Int64DataFrameColumn.cs

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Text;
8+
9+
namespace Microsoft.Data.Analysis
10+
{
11+
public partial class Int64DataFrameColumn : PrimitiveDataFrameColumn<long>
12+
{
13+
public Int64DataFrameColumn(string name, IEnumerable<long?> values) : base(name, values) { }
14+
15+
public Int64DataFrameColumn(string name, IEnumerable<long> values) : base(name, values) { }
16+
17+
public Int64DataFrameColumn(string name, long length = 0) : base(name, length) { }
18+
19+
public Int64DataFrameColumn(string name, ReadOnlyMemory<byte> buffer, ReadOnlyMemory<byte> nullBitMap, int length = 0, int nullCount = 0) : base(name, buffer, nullBitMap, length, nullCount) { }
20+
21+
internal Int64DataFrameColumn(string name, PrimitiveColumnContainer<long> values) : base(name, values) { }
22+
23+
protected override PrimitiveDataFrameColumn<long> CreateNewColumn(string name, long length = 0)
24+
{
25+
return new Int64DataFrameColumn(name, length);
26+
}
27+
28+
internal override PrimitiveDataFrameColumn<long> CreateNewColumn(string name, PrimitiveColumnContainer<long> container)
29+
{
30+
return new Int64DataFrameColumn(name, container);
31+
}
32+
}
33+
}

‎src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-

1+
22

33
// Licensed to the .NET Foundation under one or more agreements.
44
// The .NET Foundation licenses this file to you under the MIT license.

‎src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
using System.Text.Json.Serialization;
1111
using Microsoft.Extensions.DependencyInjection;
1212
using Microsoft.ML.Runtime;
13+
using Microsoft.ML.SearchSpace.Option;
1314
using Newtonsoft.Json;
1415
using static Microsoft.ML.DataOperationsCatalog;
1516

@@ -24,14 +25,18 @@ public static class AutoMLExperimentExtension
2425
/// <param name="experiment"><see cref="AutoMLExperiment"/></param>
2526
/// <param name="train">dataset for training a model.</param>
2627
/// <param name="validation">dataset for validating a model during training.</param>
28+
/// <param name="subSamplingTrainDataset">determine if subsampling <paramref name="train"/> to train. This will be useful if <paramref name="train"/> is too large to be held in memory.</param>
2729
/// <returns><see cref="AutoMLExperiment"/></returns>
28-
public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView validation)
30+
public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView validation, bool subSamplingTrainDataset = false)
2931
{
30-
var datasetManager = new TrainValidateDatasetManager()
32+
var datasetManager = new TrainValidateDatasetManager(train, validation);
33+
34+
if (subSamplingTrainDataset)
3135
{
32-
TrainDataset = train,
33-
ValidateDataset = validation
34-
};
36+
var searchSpace = new SearchSpace.SearchSpace();
37+
searchSpace.Add(datasetManager.SubSamplingKey, new UniformSingleOption(0, 1, false, 0.1f));
38+
experiment.AddSearchSpace(nameof(TrainValidateDatasetManager), searchSpace);
39+
}
3540

3641
experiment.ServiceCollection.AddSingleton<IDatasetManager>(datasetManager);
3742
experiment.ServiceCollection.AddSingleton(datasetManager);
@@ -62,13 +67,7 @@ public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, Trai
6267
/// <returns><see cref="AutoMLExperiment"/></returns>
6368
public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView dataset, int fold = 10, string samplingKeyColumnName = null)
6469
{
65-
var datasetManager = new CrossValidateDatasetManager()
66-
{
67-
Dataset = dataset,
68-
Fold = fold,
69-
SamplingKeyColumnName = samplingKeyColumnName,
70-
};
71-
70+
var datasetManager = new CrossValidateDatasetManager(dataset, fold, samplingKeyColumnName);
7271
experiment.ServiceCollection.AddSingleton<IDatasetManager>(datasetManager);
7372
experiment.ServiceCollection.AddSingleton(datasetManager);
7473

‎src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,7 @@ public TrialResult Run(TrialSettings settings)
391391
{
392392
var stopWatch = new Stopwatch();
393393
stopWatch.Start();
394-
var fold = datasetManager.Fold ?? 5;
394+
var fold = datasetManager.Fold;
395395
var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetManager.Dataset, pipeline, fold, metricManager.LabelColumn);
396396

397397
// now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best.
@@ -420,8 +420,8 @@ public TrialResult Run(TrialSettings settings)
420420
{
421421
var stopWatch = new Stopwatch();
422422
stopWatch.Start();
423-
var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);
424-
var eval = model.Transform(trainTestDatasetManager.ValidateDataset);
423+
var model = pipeline.Fit(trainTestDatasetManager.LoadTrainDataset(_context, settings));
424+
var eval = model.Transform(trainTestDatasetManager.LoadValidateDataset(_context, settings));
425425
var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn);
426426
var metric = GetMetric(metricManager.Metric, metrics);
427427
var loss = metricManager.IsMaximize ? -metric : metric;

‎src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -369,7 +369,7 @@ public TrialResult Run(TrialSettings settings)
369369
{
370370
var stopWatch = new Stopwatch();
371371
stopWatch.Start();
372-
var fold = datasetManager.Fold ?? 5;
372+
var fold = datasetManager.Fold;
373373
var metrics = _context.MulticlassClassification.CrossValidate(datasetManager.Dataset, pipeline, fold, metricManager.LabelColumn);
374374

375375
// now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best.
@@ -398,8 +398,8 @@ public TrialResult Run(TrialSettings settings)
398398
{
399399
var stopWatch = new Stopwatch();
400400
stopWatch.Start();
401-
var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);
402-
var eval = model.Transform(trainTestDatasetManager.ValidateDataset);
401+
var model = pipeline.Fit(trainTestDatasetManager.LoadTrainDataset(_context, settings));
402+
var eval = model.Transform(trainTestDatasetManager.LoadValidateDataset(_context, settings));
403403
var metrics = _context.MulticlassClassification.Evaluate(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn);
404404
var metric = GetMetric(metricManager.Metric, metrics);
405405
var loss = metricManager.IsMaximize ? -metric : metric;

‎src/Microsoft.ML.AutoML/API/RegressionExperiment.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
396396
{
397397
var stopWatch = new Stopwatch();
398398
stopWatch.Start();
399-
var fold = datasetManager.Fold ?? 5;
399+
var fold = datasetManager.Fold;
400400
var metrics = _context.Regression.CrossValidate(datasetManager.Dataset, pipeline, fold, metricManager.LabelColumn);
401401

402402
// now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best.
@@ -425,8 +425,8 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
425425
{
426426
var stopWatch = new Stopwatch();
427427
stopWatch.Start();
428-
var model = pipeline.Fit(trainTestDatasetManager.TrainDataset);
429-
var eval = model.Transform(trainTestDatasetManager.ValidateDataset);
428+
var model = pipeline.Fit(trainTestDatasetManager.LoadTrainDataset(_context, settings));
429+
var eval = model.Transform(trainTestDatasetManager.LoadValidateDataset(_context, settings));
430430
var metrics = _context.Regression.Evaluate(eval, metricManager.LabelColumn, scoreColumnName: metricManager.ScoreColumn);
431431
var metric = GetMetric(metricManager.Metric, metrics);
432432
var loss = metricManager.IsMaximize ? -metric : metric;

0 commit comments

Comments
 (0)