Fix broken tests and VS solutions caused by source code reorganization and other build infra cleanup

Changes:
* Fix broken tests and VS solutions caused by source code reorganization
* Add a check to validate generated code and solutions on PRs
* backport some source code reorg to src/Identity
* Fix startvs.cmd if you've already run build.ps1
* Add PR checks for tests on Linux/macOS
* Skip broken Nginx tests
* Add conditions to skip IIS tests on non-Windows platforms

Author: natemcmaster

.azure/pipelines/ci.yml

@@ -16,3 +16,13 @@ jobs:
     beforeBuild:
     - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1"
       displayName: Setup IISExpress test certificates
+- template: jobs/default-build.yml
+  parameters:
+    jobName: macOs_Build
+    jobDisplayName: "Build and test : macOS"
+    agentOs: macOS
+- template: jobs/default-build.yml
+  parameters:
+    jobName: Linux_Build
+    jobDisplayName: "Build and test : Linux"
+    agentOs: Linux

.azure/pipelines/pr-validation-temp.yml

@@ -28,3 +28,11 @@ jobs:
     jobDisplayName: "Build only : Linux"
     agentOs: Linux
     buildArgs: '/p:SkipTests=true'
+- job: Code_check
+  displayName: Code check
+  workspace:
+    clean: all
+  pool:
+    vmImage: vs2017-win2016
+  steps:
+  - powershell: ./eng/scripts/CodeCheck.ps1 -ci

Directory.Build.props

@@ -38,6 +38,8 @@
 
   <PropertyGroup>
     <StandardTestTfms>netcoreapp2.1;net461</StandardTestTfms>
+    <!-- Projects which reference Microsoft.AspNetCore.Mvc.Testing should import this targets file to ensure dependency .deps.json files are copied into test output. -->
+    <MvcTestingTargets>$(MSBuildThisFileDirectory)src\Mvc\src\Microsoft.AspNetCore.Mvc.Testing\build\netstandard2.0\Microsoft.AspNetCore.Mvc.Testing.targets</MvcTestingTargets>
   </PropertyGroup>
 
   <Import Project="eng\Dependencies.props" />

build/repo.props

@@ -63,13 +63,10 @@
         <ProjectToExclude Include="
                           $(RepositoryRoot)src\Servers\HttpSys\samples\TestClient\TestClient.csproj;
                           $(RepositoryRoot)src\Middleware\WebSockets\samples\TestServer\TestServer.csproj;
+                          $(RepositoryRoot)src\Identity\samples\NativeWPFClient\NativeWPFClient.csproj;
                           "
                           Condition=" '$(MSBuildRuntimeType)' == 'Core' " />
 
-
-    <!-- Exclude the WPF samples for now because they use classic .csproj, which is not yet supported in our build. -->
-    <ProjectToExclude Include="$(RepositoryRoot)src\Identity\Identity\samples\NativeWPFClient\NativeWPFClient.csproj;" />
-
         <!-- Exclude the websockets samples for now because they use classic .csproj, which is not yet supported in our build. -->
         <ProjectToExclude Include="
                           $(RepositoryRoot)src\Middleware\WebSockets\samples\**\*.csproj;

eng/ProjectReferences.props

@@ -35,12 +35,12 @@
     <ProjectReferenceProvider Include="Microsoft.AspNetCore.Routing" ProjectPath="$(RepositoryRoot)src\Http\Routing\src\Microsoft.AspNetCore.Routing.csproj" />
     <ProjectReferenceProvider Include="Microsoft.AspNetCore.WebUtilities" ProjectPath="$(RepositoryRoot)src\Http\WebUtilities\src\Microsoft.AspNetCore.WebUtilities.csproj" />
     <ProjectReferenceProvider Include="Microsoft.AspNetCore.Html.Abstractions" ProjectPath="$(RepositoryRoot)src\Html\Abstractions\src\Microsoft.AspNetCore.Html.Abstractions.csproj" />
-    <ProjectReferenceProvider Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" ProjectPath="$(RepositoryRoot)src\Identity\EF\src\Microsoft.AspNetCore.Identity.EntityFrameworkCore.csproj" />
-    <ProjectReferenceProvider Include="Microsoft.Extensions.Identity.Core" ProjectPath="$(RepositoryRoot)src\Identity\Extensions\Core\src\Microsoft.Extensions.Identity.Core.csproj" />
-    <ProjectReferenceProvider Include="Microsoft.Extensions.Identity.Stores" ProjectPath="$(RepositoryRoot)src\Identity\Extensions\Stores\src\Microsoft.Extensions.Identity.Stores.csproj" />
-    <ProjectReferenceProvider Include="Microsoft.AspNetCore.Identity" ProjectPath="$(RepositoryRoot)src\Identity\Identity\Core\src\Microsoft.AspNetCore.Identity.csproj" />
-    <ProjectReferenceProvider Include="Microsoft.AspNetCore.Identity.Specification.Tests" ProjectPath="$(RepositoryRoot)src\Identity\Identity\Specification.Tests\src\Microsoft.AspNetCore.Identity.Specification.Tests.csproj" />
-    <ProjectReferenceProvider Include="Microsoft.AspNetCore.Identity.UI" ProjectPath="$(RepositoryRoot)src\Identity\Identity\UI\src\Microsoft.AspNetCore.Identity.UI.csproj" />
+    <ProjectReferenceProvider Include="Microsoft.AspNetCore.Identity" ProjectPath="$(RepositoryRoot)src\Identity\Core\src\Microsoft.AspNetCore.Identity.csproj" />
+    <ProjectReferenceProvider Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" ProjectPath="$(RepositoryRoot)src\Identity\EntityFrameworkCore\src\Microsoft.AspNetCore.Identity.EntityFrameworkCore.csproj" />
+    <ProjectReferenceProvider Include="Microsoft.Extensions.Identity.Core" ProjectPath="$(RepositoryRoot)src\Identity\Extensions.Core\src\Microsoft.Extensions.Identity.Core.csproj" />
+    <ProjectReferenceProvider Include="Microsoft.Extensions.Identity.Stores" ProjectPath="$(RepositoryRoot)src\Identity\Extensions.Stores\src\Microsoft.Extensions.Identity.Stores.csproj" />
+    <ProjectReferenceProvider Include="Microsoft.AspNetCore.Identity.Specification.Tests" ProjectPath="$(RepositoryRoot)src\Identity\Specification.Tests\src\Microsoft.AspNetCore.Identity.Specification.Tests.csproj" />
+    <ProjectReferenceProvider Include="Microsoft.AspNetCore.Identity.UI" ProjectPath="$(RepositoryRoot)src\Identity\UI\src\Microsoft.AspNetCore.Identity.UI.csproj" />
     <ProjectReferenceProvider Include="Microsoft.AspNetCore.Connections.Abstractions" ProjectPath="$(RepositoryRoot)src\Servers\Connections.Abstractions\src\Microsoft.AspNetCore.Connections.Abstractions.csproj" />
     <ProjectReferenceProvider Include="Microsoft.AspNetCore.Server.HttpSys" ProjectPath="$(RepositoryRoot)src\Servers\HttpSys\src\Microsoft.AspNetCore.Server.HttpSys.csproj" />
     <ProjectReferenceProvider Include="Microsoft.AspNetCore.Server.IISIntegration" ProjectPath="$(RepositoryRoot)src\Servers\IIS\IISIntegration\src\Microsoft.AspNetCore.Server.IISIntegration.csproj" />

eng/scripts/CodeCheck.ps1

@@ -0,0 +1,69 @@
+#requires -version 5
+<#
+.SYNOPSIS
+This script runs a quick check for common errors, such as checking that Visual Studio solutions are up to date or that generated code has been committed to source.
+#>
+param(
+    [switch]$ci
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 1
+Import-Module -Scope Local -Force "$PSScriptRoot/common.psm1"
+
+$repoRoot = Resolve-Path "$PSScriptRoot/../../"
+
+[string[]] $errors = @()
+
+try {
+    #
+    # Solutions
+    #
+
+    Write-Host "Checking that solutions are up to date"
+
+    Get-ChildItem "$repoRoot/*.sln" -Recurse | % {
+        Write-Host "  Checking $(Split-Path -Leaf $_)"
+        $slnDir = Split-Path -Parent $_
+        $sln = $_
+        & dotnet sln $_ list `
+            | ? { $_ -ne 'Project(s)' -and $_ -ne '----------' } `
+            | % {
+                $proj = Join-Path $slnDir $_
+                if (-not (Test-Path $proj)) {
+                    $errors += "Missing project. Solution references a project which does not exist: $proj. [$sln] "
+                }
+            }
+        }
+
+    #
+    # Generated code check
+    #
+
+    Write-Host "Re-running code generation"
+
+    Invoke-Block {
+        [string[]] $generateArgs = @()
+        if ($ci) {
+            $generateArgs += '-ci'
+        }
+        & $repoRoot/build.cmd /t:GenerateProjectList @generateArgs
+    }
+
+    Write-Host "git diff"
+    & git diff --ignore-space-at-eol --exit-code
+    if ($LastExitCode -ne 0) {
+        $status = git status -s | Out-String
+        $status = $status -replace "`n","`n    "
+        $errors += "Generated code is not up to date."
+    }
+}
+finally {
+    foreach ($err in $errors) {
+        Write-Host -f Red "error : $err"
+    }
+
+    if ($errors) {
+        exit 1
+    }
+}

eng/scripts/common.psm1

@@ -0,0 +1,74 @@
+$ErrorActionPreference = 'Stop'
+# Update the default TLS support to 1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+function Invoke-Block([scriptblock]$cmd, [string]$WorkingDir = $null) {
+    if ($WorkingDir) {
+        Push-Location $WorkingDir
+    }
+
+    try {
+
+        $cmd | Out-String | Write-Verbose
+        & $cmd
+
+        # Need to check both of these cases for errors as they represent different items
+        # - $?: did the powershell script block throw an error
+        # - $lastexitcode: did a windows command executed by the script block end in error
+        if ((-not $?) -or ($lastexitcode -ne 0)) {
+            if ($error -ne $null)
+            {
+                Write-Warning $error[0]
+            }
+            throw "Command failed to execute: $cmd"
+        }
+    }
+    finally {
+        if ($WorkingDir) {
+            Pop-Location
+        }
+    }
+}
+
+function SaveXml([xml]$xml, [string]$path) {
+    Write-Verbose "Saving to $path"
+    $ErrorActionPreference = 'stop'
+
+    $settings = New-Object System.XML.XmlWriterSettings
+    $settings.OmitXmlDeclaration = $true
+    $settings.Encoding = New-Object System.Text.UTF8Encoding( $true )
+    $writer = [System.XML.XMLTextWriter]::Create($path, $settings)
+    $xml.Save($writer)
+    $writer.Close()
+}
+
+function LoadXml([string]$path) {
+    Write-Verbose "Reading from $path"
+
+    $ErrorActionPreference = 'stop'
+    $obj = new-object xml
+    $obj.PreserveWhitespace = $true
+    $obj.Load($path)
+    return $obj
+}
+
+function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) {
+    if ($RemotePath -notlike 'http*') {
+        Copy-Item $RemotePath $LocalPath
+        return
+    }
+
+    $retries = 10
+    while ($retries -gt 0) {
+        $retries -= 1
+        try {
+            Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath
+            return
+        }
+        catch {
+            Write-Verbose "Request failed. $retries retries remaining"
+        }
+    }
+
+    Write-Error "Download failed: '$RemotePath'."
+}

korebuild-lock.txt

@@ -1,2 +1,2 @@
-version:2.1.3-rtm-15850
-commithash:725f1523b274d40570f4c923c4712a01f2ef3387
+version:2.1.7-build-20181213.1
+commithash:69f12b27418b4925734d427b2c3e36c53f265c12

src/Azure/Azure.sln

@@ -39,7 +39,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Static
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.HttpsPolicy", "..\Middleware\HttpsPolicy\src\Microsoft.AspNetCore.HttpsPolicy.csproj", "{A3789DA9-2E0E-41D6-BDDD-11DA4D3DD72D}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Server.IISIntegration", "..\Servers\IIS\src\Microsoft.AspNetCore.Server.IISIntegration\Microsoft.AspNetCore.Server.IISIntegration.csproj", "{9631D314-7B0C-4CEC-8650-0259A8F4C77A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Server.IISIntegration", "..\Servers\IIS\IISIntegration\src\Microsoft.AspNetCore.Server.IISIntegration.csproj", "{9631D314-7B0C-4CEC-8650-0259A8F4C77A}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Server.Kestrel.Https", "..\Servers\Kestrel\Https\src\Microsoft.AspNetCore.Server.Kestrel.Https.csproj", "{01B31C53-7BAB-4DA7-A55D-C67D9B1FB4A2}"
 EndProject

src/Azure/AzureAD/test/FunctionalTests/Microsoft.AspNetCore.Authentication.AzureAD.FunctionalTests.csproj

@@ -17,6 +17,5 @@
     <Reference Include="Microsoft.Extensions.Logging.Testing" />
   </ItemGroup>
 
-
-  <Import Project="$(RepositoryRoot)src\Mvc\src\Microsoft.AspNetCore.Mvc.Testing\build\netstandard2.0\Microsoft.AspNetCore.Mvc.Testing.targets" />
+  <Import Project="$(MvcTestingTargets)" Condition="'$(MvcTestingTargets)' != ''" />
 </Project>