Optimize pipelines (#8520)

Too Many commits
microsoft · Dec 7, 2019 · a7ff238 · a7ff238
1 parent 6fb120f
commit a7ff238
Show file tree

Hide file tree

Showing 17 changed files with 580 additions and 159 deletions.
diff --git a/build/.cachesalt b/build/.cachesalt
@@ -1 +1 @@
-2019-08-30T20:24:23.714Z
+2019-12-01T02:20:58.491Z
diff --git a/build/azure-pipelines/darwin/sql-product-build-darwin.yml b/build/azure-pipelines/darwin/sql-product-build-darwin.yml
@@ -1,4 +1,24 @@
 steps:
+  - script: |
+      mkdir -p .build
+      echo -n $BUILD_SOURCEVERSION > .build/commit
+      echo -n $VSCODE_QUALITY > .build/quality
+    displayName: Prepare cache flag
+
+  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
+    inputs:
+      keyfile: 'build/.cachesalt, .build/commit, .build/quality'
+      targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
+      vstsFeed: 'BuildCache'
+      platformIndependent: true
+      alias: 'Compilation'
+
+  - script: |
+      set -e
+      exit 1
+    displayName: Check RestoreCache
+    condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
+
   - task: NodeTool@0
     inputs:
       versionSpec: '10.15.3'
@@ -7,12 +27,6 @@ steps:
     inputs:
       versionSpec: '1.x'
 
-  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
-    inputs:
-      keyfile: '.yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
-      targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
-      vstsFeed: 'BuildCache'
-
   - task: AzureKeyVault@1
     displayName: 'Azure Key Vault: Get Secrets'
     inputs:
@@ -40,6 +54,12 @@ steps:
       git merge $(node -p "require('./package.json').distro")
     displayName: Merge distro
 
+  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
+    inputs:
+      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
+      targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
+      vstsFeed: 'BuildCache'
+
   - script: |
       set -e
       CHILD_CONCURRENCY=1 yarn --frozen-lockfile
@@ -48,7 +68,7 @@ steps:
 
   - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
     inputs:
-      keyfile: '.yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
+      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
       targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
       vstsFeed: 'BuildCache'
     condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
@@ -71,8 +91,13 @@ steps:
 
   - script: |
       set -e
-      yarn gulp vscode-darwin-min
+      yarn gulp package-rebuild-extensions
+      yarn gulp vscode-darwin-min-ci
+      yarn gulp vscode-reh-darwin-min-ci
+      yarn gulp vscode-reh-web-darwin-min-ci
     displayName: Build
+    env:
+      VSCODE_MIXIN_PASSWORD: $(github-distro-mixin-password)
 
   - task: ArchiveFiles@2 # WHY ARE WE DOING THIS?
     displayName: 'Archive build scripts source'
@@ -140,6 +165,10 @@ steps:
     timeoutInMinutes: 20
     continueOnError: true
 
+  - script: |
+      pushd .. && mv azuredatastudio-reh-darwin azuredatastudio-server-darwin && zip -Xry $(Build.ArtifactStagingDirectory)/azuredatastudio-server-darwin.zip azuredatastudio-server-darwin && popd
+    displayName: 'Package server'
+
   - script: | # WHY ARE WE DOING THIS?
       set -e
       PACKAGEJSON=`ls $(Build.SourcesDirectory)/package.json`

diff --git a/build/azure-pipelines/docker/Dockerfile b/build/azure-pipelines/docker/Dockerfile
@@ -0,0 +1,16 @@
+#Download base image ubuntu 16.04
+FROM ubuntu:16.04
+
+# Update Software repository
+RUN apt-get update
+
+RUN apt-get install -y libxkbfile-dev pkg-config libsecret-1-dev libxss1 dbus libgtk-3-0
+
+ADD ./ /opt/ads-server
+
+RUN chmod +x /opt/ads-server/server.sh && chmod +x /opt/ads-server/node
+
+CMD ["/opt/ads-server/server.sh"]
+
+EXPOSE 8000:8000
+EXPOSE 8001:8001
diff --git a/build/azure-pipelines/linux/Dockerfile b/build/azure-pipelines/linux/Dockerfile
@@ -0,0 +1,20 @@
+#Download base image ubuntu 16.04
+FROM ubuntu:16.04
+
+# Update Software repository
+RUN apt-get update --fix-missing
+
+RUN apt-get install -y libxkbfile-dev pkg-config libsecret-1-dev libxss1 dbus xvfb libgtk-3-0 \
+	libkrb5-dev git apt-transport-https ca-certificates curl gnupg-agent software-properties-common \
+	libnss3 libasound2 make gcc libx11-dev fakeroot rpm
+
+#docker
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+RUN apt-key fingerprint 0EBFCD88
+RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+RUN apt-get update
+RUN apt-get -y install docker-ce docker-ce-cli containerd.io
+
+ADD ./xvfb.init /etc/init.d/xvfb
+RUN chmod +x /etc/init.d/xvfb
+RUN update-rc.d xvfb defaults
diff --git a/build/azure-pipelines/linux/sql-product-build-linux.yml b/build/azure-pipelines/linux/sql-product-build-linux.yml
@@ -1,17 +1,27 @@
 steps:
-  - task: NodeTool@0
+  - script: |
+      mkdir -p .build
+      echo -n $BUILD_SOURCEVERSION > .build/commit
+      echo -n $VSCODE_QUALITY > .build/quality
+    displayName: Prepare cache flag
+
+  - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
     inputs:
-      versionSpec: '10.15.1'
+      keyfile: 'build/.cachesalt, .build/commit, .build/quality'
+      targetfolder: '.build, out-build, out-vscode-min, out-vscode-reh-min, out-vscode-reh-web-min'
+      vstsFeed: 'BuildCache'
+      platformIndependent: true
+      alias: 'Compilation'
 
   - script: |
       set -e
-      sudo apt-get update
-      sudo apt-get install -y libxkbfile-dev pkg-config libsecret-1-dev libxss1 dbus xvfb libgtk-3-0 libkrb5-dev #{{SQL CARBON EDIT}} add kerberos dep
-      sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb
-      sudo chmod +x /etc/init.d/xvfb
-      sudo update-rc.d xvfb defaults
-      sudo service xvfb start
-    displayName: 'System Installs'
+      exit 1
+    displayName: Check RestoreCache
+    condition: and(succeeded(), ne(variables['CacheRestored-Compilation'], 'true'))
+
+  - task: NodeTool@0
+    inputs:
+      versionSpec: '10.15.1'
 
   - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@3
     inputs:
@@ -45,7 +55,7 @@ steps:
 
   - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1
     inputs:
-      keyfile: '.yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
+      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
       targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
       vstsFeed: 'BuildCache'
 
@@ -57,7 +67,7 @@ steps:
 
   - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1
     inputs:
-      keyfile: '.yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
+      keyfile: 'build/.cachesalt, .yarnrc, remote/.yarnrc, **/yarn.lock, !**/node_modules/**/yarn.lock, !**/.*/**/yarn.lock, !samples/**/yarn.lock'
       targetfolder: '**/node_modules, !**/node_modules/**/node_modules, !samples/**/node_modules'
       vstsFeed: 'BuildCache'
     condition: and(succeeded(), ne(variables['CacheRestored'], 'true'))
@@ -76,25 +86,28 @@ steps:
   - script: |
       set -e
       yarn gulp install-sqltoolsservice
-    displayName: Install sqltoolsservice
-
-  - script: |
-      set -e
       yarn gulp install-ssmsmin
-    displayName: Install ssmsmin
+    displayName: Install extension binaries
 
   - script: |
       set -e
-      yarn gulp vscode-linux-x64-min
+      yarn gulp vscode-linux-x64-min-ci
+      yarn gulp vscode-reh-linux-x64-min-ci
+      yarn gulp vscode-reh-web-linux-x64-min-ci
     displayName: Build
+    env:
+      VSCODE_MIXIN_PASSWORD: $(github-distro-mixin-password)
 
   - script: |
       set -e
-      yarn gulp compile-extensions
-    displayName: Compile Extensions
+      service xvfb start
+    displayName: Start xvfb
+    condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 
   - script: |
       set -e
+      yarn gulp package-rebuild-extensions
+      yarn gulp compile-extensions
       yarn gulp package-external-extensions
     displayName: Package External extensions
 
@@ -111,11 +124,19 @@ steps:
       PathtoPublish: '$(Build.BinariesDirectory)/source.tar.gz'
       ArtifactName: source
 
-  - script: DISPLAY=:10 ./scripts/test-extensions-unit.sh
+  - script: |
+      APP_ROOT=$(agent.builddirectory)/azuredatastudio-linux-x64
+      APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
+      INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
+      DISPLAY=:10 ./scripts/test-extensions-unit.sh
     displayName: 'Run Stable Extension Unit Tests'
     condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
 
-  - script: DISPLAY=:10 ./scripts/test-extensions-unit-unstable.sh
+  - script: |
+      APP_ROOT=$(agent.builddirectory)/azuredatastudio-linux-x64
+      APP_NAME=$(node -p "require(\"$APP_ROOT/resources/app/product.json\").applicationName")
+      INTEGRATION_TEST_ELECTRON_PATH="$APP_ROOT/$APP_NAME" \
+      DISPLAY=:10 ./scripts/test-extensions-unit-unstable.sh
     displayName: 'Run Unstable Extension Unit Tests'
     continueOnError: true
     condition: and(succeeded(), eq(variables['RUN_UNSTABLE_TESTS'], 'true'))
@@ -157,6 +178,20 @@ steps:
       SourceFolder: '$(Build.SourcesDirectory)/../vsix'
       TargetFolder: '$(Build.ArtifactStagingDirectory)/vsix'
 
+  - script: |
+      set -e
+      docker build -t azuredatastudio-server -f build/azure-pipelines/docker/Dockerfile $(agent.builddirectory)/azuredatastudio-reh-linux-x64
+      docker save azuredatastudio-server | gzip > azuredatastudio-server-docker.tar.gz
+      cp azuredatastudio-server-docker.tar.gz $(Build.ArtifactStagingDirectory)
+    displayName: "Create docker image"
+
+  - script: |
+      set -e
+      cd $(agent.builddirectory)
+      tar --owner=0 --group=0 -czf azuredatastudio-server-linux-x64.tar.gz azuredatastudio-reh-linux-x64
+      cp azuredatastudio-server-linux-x64.tar.gz $(Build.ArtifactStagingDirectory)
+    displayName: 'Package server'
+
   - script: | # WHY ARE WE DOING THIS?
       set -e
       PACKAGEJSON="$(Build.SourcesDirectory)/package.json"

diff --git a/build/azure-pipelines/sql-product-build.yml b/build/azure-pipelines/sql-product-build.yml
@@ -1,30 +1,54 @@
+resources:
+  containers:
+  - container: linux-x64
+    image: sqltoolscontainers.azurecr.io/linux-build-agent:x64
+    endpoint: ContainerRegistry
+
 jobs:
+- job: Compile
+  pool:
+    vmImage: 'Ubuntu-16.04'
+  container: linux-x64
+  steps:
+  - template: sql-product-compile.yml
+
 - job: macOS
   condition: eq(variables['VSCODE_BUILD_MACOS'], 'true')
   pool:
     vmImage: macOS 10.13
+  dependsOn:
+  - Compile
   steps:
   - template: darwin/sql-product-build-darwin.yml
-  timeoutInMinutes: 90
-  cancelTimeoutInMinutes: 5
 
 - job: Linux
   condition: eq(variables['VSCODE_BUILD_LINUX'], 'true')
   pool:
     vmImage: 'Ubuntu-16.04'
+  container: linux-x64
+  dependsOn:
+  - Compile
   steps:
   - template: linux/sql-product-build-linux.yml
-  timeoutInMinutes: 90
-  cancelTimeoutInMinutes: 5
-
+
 - job: Windows
   condition: eq(variables['VSCODE_BUILD_WIN32'], 'true')
   pool:
-    name: mssqltools
+    vmImage: VS2017-Win2016
+  dependsOn:
+  - Compile
   steps:
   - template: win32/sql-product-build-win32.yml
-  timeoutInMinutes: 90
-  cancelTimeoutInMinutes: 5
+
+- job: Windows_Test
+  condition: and(succeeded(), eq(variables['VSCODE_BUILD_WIN32'], 'true'))
+  pool:
+    name: mssqltools
+  dependsOn:
+  - Linux
+  - Windows
+  steps:
+  - template: win32/sql-product-test-win32.yml
 
 - job: Release
   condition: and(succeeded(), or(eq(variables['VSCODE_RELEASE'], 'true'), and(eq(variables['VSCODE_QUALITY'], 'insider'), eq(variables['Build.Reason'], 'Schedule'))))
@@ -34,6 +58,7 @@ jobs:
   - macOS
   - Linux
   - Windows
+  - Windows_Test
   steps:
   - template: sql-release.yml