Build all corefx packages (#653)

* Build all corefx packages

CoreFX includes a number of packages that are not built when using
source-build. For example, Microsoft.NETCore.Platforms is not built. It
is consumed via NuGet. This causes two issues:

1. The bits included in the final cli build are not all compiled from
source. This goes against overall goal of source-build.

3. Any changes made to such packages in corefx do not flow through. As
an example, this means additional runtime IDs added to
Microsoft.NETCore.Platforms are not included in the final built cli.

Fix that by passing a flag to indicate all of CoreFX should be built.

Fixes #650

* Add separate folder for previously source-built packages that are used earlier in the build than they are produced.

* Update the prebuilt baseline.

Author: omajid

build-source-tarball.sh

@@ -110,6 +110,7 @@ rm -rf $TARBALL_ROOT/Tools/configuration/configuration.props
 cp $SCRIPT_ROOT/support/tarball/build.sh $TARBALL_ROOT/build.sh
 
 mkdir -p $TARBALL_ROOT/prebuilt/nuget-packages
+mkdir -p $TARBALL_ROOT/prebuilt/source-built
 find $SCRIPT_ROOT/packages -name '*.nupkg' -exec cp {} $TARBALL_ROOT/prebuilt/nuget-packages/ \;
 find $SCRIPT_ROOT/bin/obj/x64/Release/nuget-packages -name '*.nupkg' -exec cp {} $TARBALL_ROOT/prebuilt/nuget-packages/ \;
 
@@ -129,6 +130,13 @@ do
     fi
 done
 
+echo 'Copying source-built packages to tarball to replace packages needed before they are built...'
+
+for built_package in $(find $SCRIPT_ROOT/bin/obj/x64/Release/blob-feed/packages/ -name '*.nupkg')
+do
+    cp $built_package $TARBALL_ROOT/prebuilt/source-built/
+done
+
 echo 'WORKAROUND: Overwriting the source-built roslyn-tools MSBuild files with prebuilt so that roslyn-tools can successfully build in the tarball... (https://github.com/dotnet/source-build/issues/654)'
 
 ROSLYN_TOOLS_PACKAGE='RoslynTools.RepoToolset'

dir.props

@@ -57,6 +57,7 @@
     <SourceBuiltPackagesPath>$(SourceBuiltBlobFeedDir)packages/</SourceBuiltPackagesPath>
     <SourceBuiltAssetsDir>$(SourceBuiltBlobFeedDir)assets/</SourceBuiltAssetsDir>
     <PrebuiltPackagesPath>$(ProjectDir)prebuilt/nuget-packages/</PrebuiltPackagesPath>
+    <PrebuiltSourceBuiltPackagesPath>$(ProjectDir)prebuilt/source-built/</PrebuiltSourceBuiltPackagesPath>
     <SourceBuiltTarBallPath>$(OutputPath)</SourceBuiltTarBallPath>
     <TargetInfoProps>$(IntermediatePath)TargetInfo.props</TargetInfoProps>
     <RestoreSourcePropsPath>$(IntermediatePath)RestoreSources.props</RestoreSourcePropsPath>

repos/corefx.proj

@@ -9,7 +9,7 @@
     <OverrideTargetRid Condition="'$(TargetOS)' == 'OSX'">osx-x64</OverrideTargetRid>
 
     <BuildArguments>-$(Configuration) -buildArch=$(Platform) -portable=$(OverridePortableBuild) -BuildTests=false -PackageRid=$(OverrideTargetRid)</BuildArguments>
-    <BuildCommand>$(ProjectDirectory)/build$(ShellExtension) $(BuildArguments) -- /bl</BuildCommand>
+    <BuildCommand>$(ProjectDirectory)/build$(ShellExtension) $(BuildArguments) -- /p:BuildAllPackages=true /bl</BuildCommand>
     <BuildCommand Condition="$(Platform.Contains('arm'))">$(ArmEnvironmentVariables) $(BuildCommand)</BuildCommand>
     <PackagesOutput>$(ProjectDirectory)/bin/packages/$(Configuration)</PackagesOutput>
     <CleanCommand>$(ProjectDirectory)/clean$(ShellExtension)</CleanCommand>

repos/dir.targets

@@ -140,6 +140,11 @@
                             SourcePath="$(PrebuiltPackagesPath)"
                             Condition="'$(OfflineBuild)' == 'true'" />
 
+    <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
+                            SourceName="previously-source-built"
+                            SourcePath="$(PrebuiltSourceBuiltPackagesPath)"
+                            Condition="'$(OfflineBuild)' == 'true'" />
+
     <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
                             SourceName="source-built"
                             SourcePath="$(SourceBuiltPackagesPath)" />
@@ -211,6 +216,7 @@
     <ItemGroup>
       <_DotNetRestoreSources Include="$(SourceBuiltPackagesPath)" />
       <_DotNetRestoreSources Include="$(PrebuiltPackagesPath)" Condition="'$(OfflineBuild)' == 'true'"/>
+      <_DotNetRestoreSources Include="$(PrebuiltSourceBuiltPackagesPath)" Condition="'$(OfflineBuild)' == 'true'"/>
     </ItemGroup>
     <WriteRestoreSourceProps RestoreSources="@(_DotNetRestoreSources)"
                              OutputPath="$(RestoreSourcePropsPath)" />

tools-local/prebuilt-baseline-offline.xml

@@ -176,15 +176,13 @@
     <Usage Id="Microsoft.NETCore.Platforms" Version="2.1.0-preview1-26116-01" />
     <Usage Id="Microsoft.NETCore.Platforms" Version="2.1.0-preview2-26406-04" />
     <Usage Id="Microsoft.NETCore.Platforms" Version="2.1.1-servicing-26722-02" />
-    <Usage Id="Microsoft.NETCore.Platforms" Version="2.1.1" />
     <Usage Id="Microsoft.NETCore.Runtime.CoreCLR" Version="1.0.5" />
     <Usage Id="Microsoft.NETCore.Runtime.CoreCLR" Version="1.0.7" />
     <Usage Id="Microsoft.NETCore.Runtime.CoreCLR" Version="1.1.1" />
     <Usage Id="Microsoft.NETCore.Runtime.CoreCLR" Version="1.1.2" />
     <Usage Id="Microsoft.NETCore.Targets" Version="1.0.1" />
     <Usage Id="Microsoft.NETCore.Targets" Version="1.0.3" />
     <Usage Id="Microsoft.NETCore.Targets" Version="1.1.0" />
-    <Usage Id="Microsoft.NETCore.Targets" Version="2.0.0" />
     <Usage Id="Microsoft.NETCore.Targets" Version="2.1.0-preview2-26406-04" />
     <Usage Id="Microsoft.NETCore.Windows.ApiSets" Version="1.0.1" />
     <Usage Id="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0-alpha-003" />
@@ -711,4 +709,4 @@
     <Usage Id="xunit.runner.visualstudio" Version="2.3.0" />
     <Usage Id="xunit.runner.visualstudio" Version="2.3.1" />
   </Usages>
-</UsageData>
+</UsageData>

tools-local/prebuilt-baseline-online.xml

@@ -168,7 +168,6 @@
     <Usage Id="Microsoft.NETCore.Platforms" Version="2.1.0-preview1-26116-01" />
     <Usage Id="Microsoft.NETCore.Platforms" Version="2.1.0-preview2-26406-04" />
     <Usage Id="Microsoft.NETCore.Platforms" Version="2.1.1-servicing-26722-02" />
-    <Usage Id="Microsoft.NETCore.Platforms" Version="2.1.1" />
     <Usage Id="Microsoft.NETCore.Runtime.CoreCLR" Version="1.0.5" />
     <Usage Id="Microsoft.NETCore.Runtime.CoreCLR" Version="1.0.7" />
     <Usage Id="Microsoft.NETCore.Runtime.CoreCLR" Version="1.1.1" />
@@ -177,7 +176,6 @@
     <Usage Id="Microsoft.NETCore.Targets" Version="1.0.1" />
     <Usage Id="Microsoft.NETCore.Targets" Version="1.0.3" />
     <Usage Id="Microsoft.NETCore.Targets" Version="1.1.0" />
-    <Usage Id="Microsoft.NETCore.Targets" Version="2.0.0" />
     <Usage Id="Microsoft.NETCore.Targets" Version="2.1.0-preview2-26406-04" />
     <Usage Id="Microsoft.NETCore.Windows.ApiSets" Version="1.0.1" />
     <Usage Id="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0-alpha-003" />
@@ -709,4 +707,4 @@
     <Usage Id="xunit.runner.visualstudio" Version="2.3.0" />
     <Usage Id="xunit.runner.visualstudio" Version="2.3.1" />
   </Usages>
-</UsageData>
+</UsageData>