Implement a CI pipeline (#13003)

.pipelines/build-stage.yml

@@ -36,6 +36,16 @@ parameters:
       - x64
       - arm64
 
+  - name: pool
+    type: string
+    default: ''
+
+  - name: vsoOrg
+    type: string
+
+  - name: vsoProject
+    type: string
+
   - name: esrp
     type: object
     default:
@@ -55,13 +65,19 @@ stages:
   - stage: build
     jobs:
       - job:
+
         displayName: "Formatting & localization checks"
         timeoutInMinutes: 30
-        pool:
-          type: windows
+
         variables:
           ob_outputDirectory: '$(Build.SourcesDirectory)\out'
 
+        ${{ if eq(parameters.pool, '') }}:
+          pool: {'type': 'windows'}
+
+        ${{ else }}:
+          pool: ${{ parameters.pool }}
+
         steps:
           - script: python tools/devops/validate-localization.py localization/strings en-US
             displayName: Validate localization resources
@@ -88,8 +104,12 @@ stages:
       - job: build
         displayName: "Build WSL package"
         timeoutInMinutes: 120
-        pool:
-          type: windows
+
+        ${{ if eq(parameters.pool, '') }}:
+          pool: {'type': 'windows'}
+
+        ${{ else }}:
+          pool: ${{ parameters.pool }}
 
         variables:
           NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 60
@@ -146,7 +166,7 @@ stages:
                 displayName: "CMake ${{ platform }}"
                 inputs:
                   workingDirectory: "."
-                  cmakeArgs: . --fresh -A ${{ platform }} -DCMAKE_BUILD_TYPE=Release -DCMAKE_SYSTEM_VERSION=10.0.26100.0 -DPACKAGE_VERSION=$(version.WSL_PACKAGE_VERSION) -DWSL_NUGET_PACKAGE_VERSION=$(version.WSL_NUGET_PACKAGE_VERSION) -DSKIP_PACKAGE_SIGNING=${{ parameters.isRelease }} -DOFFICIAL_BUILD=${{ parameters.isRelease }} -DPIPELINE_BUILD_ID=$(Build.BuildId)
+                  cmakeArgs: . --fresh -A ${{ platform }} -DCMAKE_BUILD_TYPE=Release -DCMAKE_SYSTEM_VERSION=10.0.26100.0 -DPACKAGE_VERSION=$(version.WSL_PACKAGE_VERSION) -DWSL_NUGET_PACKAGE_VERSION=$(version.WSL_NUGET_PACKAGE_VERSION) -DSKIP_PACKAGE_SIGNING=${{ parameters.isRelease }} -DOFFICIAL_BUILD=${{ parameters.isRelease }} -DPIPELINE_BUILD_ID=$(Build.BuildId) -DVSO_ORG=${{ parameters.vsoOrg }} -DVSO_PROJECT=${{ parameters.vsoProject }}
 
               # This additional Restore NuGet package task is added as a workaround for WSL Settings to have its' packages restored properly.
               # Without this, building wsl settings may encounter the following error:
@@ -161,47 +181,48 @@ stages:
                     condition: and(succeeded(), eq('${{ parameters.isRelease }}', true))
                     displayName: "Build ${{ target.target }} (${{ platform }})"
 
-                  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5
-                    displayName: "Sign ${{ target.target }} (${{ platform }})"
-                    condition: and(succeeded(), eq('${{ parameters.isRelease }}', true))
-                    inputs:
-                      ConnectedServiceName: ${{ parameters.esrp.ConnectedServiceName}}
-                      signConfigType: ${{ parameters.esrp.signConfigType }}
-                      SessionTimeout: ${{ parameters.esrp.SessionTimeout }}
-                      MaxConcurrency: ${{ parameters.esrp.MaxConcurrency }}
-                      MaxRetryAttempts: ${{ parameters.esrp.MaxRetryAttempts }}
-                      ServiceEndpointUrl: ${{ parameters.esrp.ServiceEndpointUrl }}
-                      AuthAKVName: ${{ parameters.esrp.AuthAKVName }}
-                      AuthSignCertName: ${{ parameters.esrp.AuthSignCertName }}
-                      AppRegistrationClientId: ${{ parameters.esrp.AppRegistrationClientId }}
-                      AppRegistrationTenantId: ${{ parameters.esrp.AppRegistrationTenantId }}
-                      FolderPath: "bin\\${{ platform }}"
-                      Pattern: "${{ target.pattern }}"
-                      UseMSIAuthentication: true
-                      EsrpClientId: ${{ parameters.esrp.EsrpClientId }}
-                      inlineOperation: |
-                        [
-                              {
-                                  "KeyCode": "CP-230012",
-                                  "OperationCode": "SigntoolSign",
-                                  "Parameters" : {
-                                        "OpusName" : "Microsoft",
-                                        "OpusInfo" : "http://www.microsoft.com",
-                                        "FileDigest" : "/fd \"SHA256\"",
-                                        "PageHash" : "/NPH",
-                                        "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+                  - ${{ if eq(parameters.isRelease, 'true') }}:
+                      - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5
+                        displayName: "Sign ${{ target.target }} (${{ platform }})"
+                        condition: and(succeeded(), eq('${{ parameters.isRelease }}', true))
+                        inputs:
+                          ConnectedServiceName: ${{ parameters.esrp.ConnectedServiceName}}
+                          signConfigType: ${{ parameters.esrp.signConfigType }}
+                          SessionTimeout: ${{ parameters.esrp.SessionTimeout }}
+                          MaxConcurrency: ${{ parameters.esrp.MaxConcurrency }}
+                          MaxRetryAttempts: ${{ parameters.esrp.MaxRetryAttempts }}
+                          ServiceEndpointUrl: ${{ parameters.esrp.ServiceEndpointUrl }}
+                          AuthAKVName: ${{ parameters.esrp.AuthAKVName }}
+                          AuthSignCertName: ${{ parameters.esrp.AuthSignCertName }}
+                          AppRegistrationClientId: ${{ parameters.esrp.AppRegistrationClientId }}
+                          AppRegistrationTenantId: ${{ parameters.esrp.AppRegistrationTenantId }}
+                          FolderPath: "bin\\${{ platform }}"
+                          Pattern: "${{ target.pattern }}"
+                          UseMSIAuthentication: true
+                          EsrpClientId: ${{ parameters.esrp.EsrpClientId }}
+                          inlineOperation: |
+                            [
+                                  {
+                                      "KeyCode": "CP-230012",
+                                      "OperationCode": "SigntoolSign",
+                                      "Parameters" : {
+                                            "OpusName" : "Microsoft",
+                                            "OpusInfo" : "http://www.microsoft.com",
+                                            "FileDigest" : "/fd \"SHA256\"",
+                                            "PageHash" : "/NPH",
+                                            "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+                                        },
+                                        "ToolName" : "sign",
+                                        "ToolVersion" : "1.0"
                                     },
-                                    "ToolName" : "sign",
-                                    "ToolVersion" : "1.0"
-                                },
-                                {
-                                    "KeyCode" : "CP-230012",
-                                    "OperationCode" : "SigntoolVerify",
-                                    "Parameters" : {},
-                                    "ToolName" : "sign",
-                                    "ToolVersion" : "1.0"
-                                }
-                          ]
+                                    {
+                                        "KeyCode" : "CP-230012",
+                                        "OperationCode" : "SigntoolVerify",
+                                        "Parameters" : {},
+                                        "ToolName" : "sign",
+                                        "ToolVersion" : "1.0"
+                                    }
+                              ]
 
                   # Replace the intermediate wslsettings binaries file with the signed versions so that any future steps building wslsettings will use the signed versions
                   - task: PowerShell@2
@@ -230,47 +251,48 @@ stages:
                       $arch = '${{ platform }}'
                       Copy-Item -Path "bin\$arch\release\wsl.msi" -Destination "$(ob_outputDirectory)\bundle\wsl.$(version.WSL_PACKAGE_VERSION).$arch.msi"
 
-          - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5
-            displayName: "Sign the bundle"
-            condition: and(succeeded(), eq('${{ parameters.isRelease }}', true))
-            inputs:
-              ConnectedServiceName: ${{ parameters.esrp.ConnectedServiceName}}
-              signConfigType: ${{ parameters.esrp.signConfigType }}
-              SessionTimeout: ${{ parameters.esrp.SessionTimeout }}
-              MaxConcurrency: ${{ parameters.esrp.MaxConcurrency }}
-              MaxRetryAttempts: ${{ parameters.esrp.MaxRetryAttempts }}
-              ServiceEndpointUrl: ${{ parameters.esrp.ServiceEndpointUrl }}
-              AuthAKVName: ${{ parameters.esrp.AuthAKVName }}
-              AuthSignCertName: ${{ parameters.esrp.AuthSignCertName }}
-              AppRegistrationClientId: ${{ parameters.esrp.AppRegistrationClientId }}
-              AppRegistrationTenantId: ${{ parameters.esrp.AppRegistrationTenantId }}
-              FolderPath: "bundle"
-              Pattern: "*.msixbundle"
-              UseMSIAuthentication: true
-              EsrpClientId: ${{ parameters.esrp.EsrpClientId }}
-              inlineOperation: |
-                [
-                      {
-                          "KeyCode": "CP-230012",
-                          "OperationCode": "SigntoolSign",
-                          "Parameters" : {
-                                "OpusName" : "Microsoft",
-                                "OpusInfo" : "http://www.microsoft.com",
-                                "FileDigest" : "/fd \"SHA256\"",
-                                "PageHash" : "/NPH",
-                                "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+          - ${{ if eq(parameters.isRelease, 'true') }}:
+              - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5
+                displayName: "Sign the bundle"
+                condition: and(succeeded(), eq('${{ parameters.isRelease }}', true))
+                inputs:
+                  ConnectedServiceName: ${{ parameters.esrp.ConnectedServiceName}}
+                  signConfigType: ${{ parameters.esrp.signConfigType }}
+                  SessionTimeout: ${{ parameters.esrp.SessionTimeout }}
+                  MaxConcurrency: ${{ parameters.esrp.MaxConcurrency }}
+                  MaxRetryAttempts: ${{ parameters.esrp.MaxRetryAttempts }}
+                  ServiceEndpointUrl: ${{ parameters.esrp.ServiceEndpointUrl }}
+                  AuthAKVName: ${{ parameters.esrp.AuthAKVName }}
+                  AuthSignCertName: ${{ parameters.esrp.AuthSignCertName }}
+                  AppRegistrationClientId: ${{ parameters.esrp.AppRegistrationClientId }}
+                  AppRegistrationTenantId: ${{ parameters.esrp.AppRegistrationTenantId }}
+                  FolderPath: "bundle"
+                  Pattern: "*.msixbundle"
+                  UseMSIAuthentication: true
+                  EsrpClientId: ${{ parameters.esrp.EsrpClientId }}
+                  inlineOperation: |
+                    [
+                          {
+                              "KeyCode": "CP-230012",
+                              "OperationCode": "SigntoolSign",
+                              "Parameters" : {
+                                    "OpusName" : "Microsoft",
+                                    "OpusInfo" : "http://www.microsoft.com",
+                                    "FileDigest" : "/fd \"SHA256\"",
+                                    "PageHash" : "/NPH",
+                                    "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+                                },
+                                "ToolName" : "sign",
+                                "ToolVersion" : "1.0"
                             },
-                            "ToolName" : "sign",
-                            "ToolVersion" : "1.0"
-                        },
-                        {
-                            "KeyCode" : "CP-230012",
-                            "OperationCode" : "SigntoolVerify",
-                            "Parameters" : {},
-                            "ToolName" : "sign",
-                            "ToolVersion" : "1.0"
-                        }
-                  ]
+                            {
+                                "KeyCode" : "CP-230012",
+                                "OperationCode" : "SigntoolVerify",
+                                "Parameters" : {},
+                                "ToolName" : "sign",
+                                "ToolVersion" : "1.0"
+                            }
+                      ]
 
           - script: md.exe $(ob_outputDirectory)\bin\nuget
             displayName: "Create the nuget directory"
@@ -279,41 +301,42 @@ stages:
               - script: nuget.exe pack ${{ package }} -OutputDirectory $(ob_outputDirectory)\bin\nuget -NonInteractive
                 displayName: Build ${{ package }}
 
-          - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5
-            displayName: "Sign nuget packages"
-            condition: and(succeeded(), eq('${{ parameters.isRelease }}', true))
-            inputs:
-              ConnectedServiceName: ${{ parameters.esrp.ConnectedServiceName}}
-              signConfigType: ${{ parameters.esrp.signConfigType }}
-              SessionTimeout: ${{ parameters.esrp.SessionTimeout }}
-              MaxConcurrency: ${{ parameters.esrp.MaxConcurrency }}
-              MaxRetryAttempts: ${{ parameters.esrp.MaxRetryAttempts }}
-              ServiceEndpointUrl: ${{ parameters.esrp.ServiceEndpointUrl }}
-              AuthAKVName: ${{ parameters.esrp.AuthAKVName }}
-              AuthSignCertName: ${{ parameters.esrp.AuthSignCertName }}
-              AppRegistrationClientId: ${{ parameters.esrp.AppRegistrationClientId }}
-              AppRegistrationTenantId: ${{ parameters.esrp.AppRegistrationTenantId }}
-              FolderPath: '$(ob_outputDirectory)\bin\nuget'
-              Pattern: "*.nupkg"
-              UseMSIAuthentication: true
-              EsrpClientId: ${{ parameters.esrp.EsrpClientId }}
-              inlineOperation: |
-                [
-                    {
-                        "KeyCode": "CP-401405",
-                        "OperationCode": "NuGetSign",
-                        "Parameters" : {},
-                          "ToolName" : "sign",
-                          "ToolVersion" : "1.0"
-                      },
-                      {
-                          "KeyCode" : "CP-401405",
-                          "OperationCode" : "NuGetVerify",
-                          "Parameters" : {},
-                          "ToolName" : "sign",
-                          "ToolVersion" : "1.0"
-                      }
-                ]
+          - ${{ if eq(parameters.isRelease, 'true') }}:
+              - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5
+                displayName: "Sign nuget packages"
+                condition: and(succeeded(), eq('${{ parameters.isRelease }}', true))
+                inputs:
+                  ConnectedServiceName: ${{ parameters.esrp.ConnectedServiceName}}
+                  signConfigType: ${{ parameters.esrp.signConfigType }}
+                  SessionTimeout: ${{ parameters.esrp.SessionTimeout }}
+                  MaxConcurrency: ${{ parameters.esrp.MaxConcurrency }}
+                  MaxRetryAttempts: ${{ parameters.esrp.MaxRetryAttempts }}
+                  ServiceEndpointUrl: ${{ parameters.esrp.ServiceEndpointUrl }}
+                  AuthAKVName: ${{ parameters.esrp.AuthAKVName }}
+                  AuthSignCertName: ${{ parameters.esrp.AuthSignCertName }}
+                  AppRegistrationClientId: ${{ parameters.esrp.AppRegistrationClientId }}
+                  AppRegistrationTenantId: ${{ parameters.esrp.AppRegistrationTenantId }}
+                  FolderPath: '$(ob_outputDirectory)\bin\nuget'
+                  Pattern: "*.nupkg"
+                  UseMSIAuthentication: true
+                  EsrpClientId: ${{ parameters.esrp.EsrpClientId }}
+                  inlineOperation: |
+                    [
+                        {
+                            "KeyCode": "CP-401405",
+                            "OperationCode": "NuGetSign",
+                            "Parameters" : {},
+                              "ToolName" : "sign",
+                              "ToolVersion" : "1.0"
+                          },
+                          {
+                              "KeyCode" : "CP-401405",
+                              "OperationCode" : "NuGetVerify",
+                              "Parameters" : {},
+                              "ToolName" : "sign",
+                              "ToolVersion" : "1.0"
+                          }
+                    ]
 
           - powershell: |
               foreach ($arch in @("x64", "ARM64"))
@@ -323,7 +346,7 @@ stages:
                   mkdir $pdbFolder
                   foreach ($filter in @("*.pdb", "*.debug"))
                   {
-                      foreach ($e in (Get-ChildItem -Recurse -Path $binFolder -Filter $filter)) {Copy-Item -Path $e.fullname -Destination (Join-Path $pdbFolder $e.name)}
+                      foreach ($e in (Get-ChildItem -Recurse -Path $binFolder -Filter $filter)) {Move-Item -Path $e.fullname -Destination (Join-Path $pdbFolder $e.name)}
                   }
               }
 
@@ -361,7 +384,7 @@ stages:
               Move-Item -Path "tools\test\CloudTest-Setup.bat" -Destination "$(ob_outputDirectory)\testbin\CloudTest-Setup.bat"
               Move-Item -Path "tools\wsl.wprp" -Destination "$(ob_outputDirectory)\testbin\wsl.wprp"
               Move-Item -Path "test\linux\unit_tests" -Destination "$(ob_outputDirectory)\testbin\unit_tests"
-              
+
               Move-Item -Path bundle\release\* -Destination $(ob_outputDirectory)\bundle
               $TestDistroVersion = (Select-Xml -Path packages.config -XPath '/packages/package[@id=''Microsoft.WSL.TestDistro'']/@version').Node.Value
               Copy-Item "packages\Microsoft.WSL.TestDistro.$TestDistroVersion\test_distro.tar.xz" "$(ob_outputDirectory)\testbin\x64"
@@ -380,5 +403,11 @@ stages:
                 $(ob_outputDirectory)/bin/**/*.exe
                 $(ob_outputDirectory)/bin/**/*.dll
 
+          - ${{ if ne(parameters.pool, '') }}:
+            - task: PublishPipelineArtifact@1
+              inputs:
+                targetPath: $(ob_outputDirectory)
+                artifactName: $(ob_artifactBaseName)$(ob_artifactSuffix)
+
           - task: CodeQL3000Finalize@0
             condition: ${{ and(parameters.isNightly, eq(variables['Build.SourceBranch'], 'refs/heads/main'))}}

.pipelines/test-job.yml

@@ -11,6 +11,10 @@ parameters:
   - name: run
     type: boolean
 
+  - name: pool
+    type: string
+    default: ''
+
 jobs:
   - job: test_${{ parameters.branch }}_${{ parameters.version }}
     displayName: "${{ parameters.version }} tests - ${{ parameters.branch }}"
@@ -22,8 +26,11 @@ jobs:
         ob_artifactSuffix: '_test'
     timeoutInMinutes: 360
     cancelTimeoutInMinutes: 420
-    pool:
-      type: cloudtestagentless
+    ${{ if eq(parameters.pool, '') }}:
+      pool: {'type': 'cloudtestagentless'}
+
+    ${{ else }}:
+      pool: ${{ parameters.pool }}
     steps:
       - task: CloudTestServerBuildTask@2
         inputs:

.pipelines/test-stage.yml

@@ -3,12 +3,16 @@ parameters:
     type: boolean
     default: false
 
+  - name: pool
+    type: string
+    default: ''
+
   - name: versions
     type: object
     default:
       - wsl1
       - wsl2
-      
+
   - name: test_images
     type: object
     default:
@@ -20,7 +24,7 @@ parameters:
         image: 2022-datacenter-g2-2024-09-10
       - name: vb_release
         image: win10-22h2-ent-g2-2024-09-10
-      
+
       # TODO: ge_release
 
 stages:
@@ -34,4 +38,5 @@ stages:
               branch: "${{ image.name }}"
               version: ${{ version }}
               image: "${{ image.image }}"
-              run: ${{ or(not(parameters.rs_prerelease_only), eq(image.name, 'rs_prerelease')) }}
+              run: ${{ or(not(parameters.rs_prerelease_only), eq(image.name, 'rs_prerelease')) }}
+              pool: "${{ parameters.pool }}"

.pipelines/wsl-build-nightly-onebranch.yml

@@ -44,6 +44,8 @@ extends:
       parameters:
         isRelease: false
         isNightly: true
+        vsoOrg: microsoft
+        vsoProject: Microsoft.WSL
 
     - template: test-stage.yml@self
       parameters:

.pipelines/wsl-build-pr.yml

@@ -4,15 +4,15 @@ trigger:
     - master
     - release/*
 
-git:
-  fetchDepth: -1
-  fetchTags: true
-
 stages:
 - template: build-stage.yml@self
   parameters:
     isRelease: false
+    pool: 'wsl-build'
+    vsoOrg: shine-oss
+    vsoProject: wsl
 
 - template: test-stage.yml@self
   parameters:
     rs_prerelease_only: true
+    pool: server

.pipelines/wsl-build-release-onebranch.yml

@@ -52,6 +52,8 @@ extends:
         isRelease: true
         packageVersion: ${{ iif(eq(parameters.testVersion, ''), variables['Build.SourceBranchName'], parameters.testVersion) }}
         traceLoggingConfig: $(ReleaseTraceLoggingConfig)
+        vsoOrg: microsoft
+        vsoProject: Microsoft.WSL
 
     - template: test-stage.yml@self
       parameters:

cloudtest/TestMap.xml.in

@@ -5,7 +5,7 @@
     <Provider Type="PipelineArtifacts">
       <Properties>
         <Add Name="CloudTest.ProviderCustomName" Value="[drop]" />
-        <Add Name="PipelineArtifactBuildUrl" Value="https://microsoft.visualstudio.com/Microsoft.WSL/_build/results?buildId=${PIPELINE_BUILD_ID}"/>
+        <Add Name="PipelineArtifactBuildUrl" Value="https://${VSO_ORG}.visualstudio.com/${VSO_PROJECT}/_build/results?buildId=${PIPELINE_BUILD_ID}"/>
         <Add Name="PipelineArtifactName" Value="drop_wsl_build"/>
       </Properties>
     </Provider>
@@ -13,21 +13,21 @@
       <Properties>
         <Add Name="CloudTest.ProviderCustomName" Value="[test_distro]" />
         <!-- When updating the drops for this make sure the rentention is set to never expire -->
-        <Add Name="DropURL" Value="https://microsoft.artifacts.visualstudio.com/DefaultCollection/_apis/drop/drops/WSL/${CLOUDTEST_TEST_DISTRO_DROP}"/>
+        <Add Name="DropURL" Value="https://${VSO_ORG}.artifacts.visualstudio.com/_apis/drop/drops/WSL/${CLOUDTEST_TEST_DISTRO_DROP}"/>
       </Properties>
     </Provider>
     <Provider Type="VSODrop">
       <Properties>
         <Add Name="CloudTest.ProviderCustomName" Value="[test_packages]" />
         <!-- When updating the drops for this make sure the rentention is set to never expire -->
-        <Add Name="DropURL" Value="https://microsoft.artifacts.visualstudio.com/DefaultCollection/_apis/drop/drops/WSL/${CLOUDTEST_TEST_PACKAGES}"/>
+        <Add Name="DropURL" Value="https://${VSO_ORG}.artifacts.visualstudio.com/_apis/drop/drops/WSL/${CLOUDTEST_TEST_PACKAGES}"/>
       </Properties>
     </Provider>
     <Provider Type="VSODrop">
       <Properties>
         <Add Name="CloudTest.ProviderCustomName" Value="[dump_tool]" />
         <!-- When updating the drops for this make sure the rentention is set to never expire -->
-        <Add Name="DropURL" Value="https://microsoft.artifacts.visualstudio.com/DefaultCollection/_apis/drop/drops/WSL/${DUMPTOOL_DROP}"/>
+        <Add Name="DropURL" Value="https://${VSO_ORG}.artifacts.visualstudio.com/_apis/drop/drops/WSL/${DUMPTOOL_DROP}"/>
       </Properties>
     </Provider>
   </Providers>