Cloud Runner Develop - 1.0 R.C 1 (#437)

Release Candidate changeset 1 - For 1.0 Cloud Runner

.github/workflows/cloud-runner-pipeline.yml

@@ -1,11 +1,7 @@
-name: Cloud Runner
+name: Cloud Runner CI Pipeline
 
 on:
-  push: { branches: [cloud-runner-develop, main] }
-#  push: { branches: [main] }
-#  pull_request:
-#    paths-ignore:
-#      - '.github/**'
+  push: { branches: [cloud-runner-develop, cloud-runner-preview] }
 
 env:
   GKE_ZONE: 'us-central1'
@@ -21,43 +17,38 @@ env:
   AWS_DEFAULT_REGION: eu-west-2
   AWS_BASE_STACK_NAME: game-ci-github-pipelines
   CLOUD_RUNNER_BRANCH: ${{ github.ref }}
-  CLOUD_RUNNER_TESTS: true
+  CLOUD_RUNNER_DEBUG: true
+  CLOUD_RUNNER_DEBUG_TREE: true
   DEBUG: true
   UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
+  PROJECT_PATH: test-project
+  UNITY_VERSION: 2019.3.15f1
+  USE_IL2CPP: false
 
 jobs:
-  awsBuild:
-    name: AWS Fargate Build
-    if: github.event.pull_request.draft == false
+  integrationTests:
+    name: Integration Tests
+    if: github.event.event_type != 'pull_request_target'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        projectPath:
-          - test-project
-        unityVersion:
-          #  - 2019.2.11f1
-          - 2019.3.15f1
-        targetPlatform:
-          #- StandaloneOSX # Build a macOS standalone (Intel 64-bit).
-          - StandaloneWindows64 # Build a Windows 64-bit standalone.
-          - StandaloneLinux64 # Build a Linux 64-bit standalone.
-          - WebGL # WebGL.
-          #- iOS # Build an iOS player.
-          #- Android # Build an Android .apk.
-          #          - StandaloneWindows # Build a Windows standalone.
-          #          - WSAPlayer # Build an Windows Store Apps player.
-          #          - PS4 # Build a PS4 Standalone.
-          #          - XboxOne # Build a Xbox One Standalone.
-          #          - tvOS # Build to Apple's tvOS platform.
-          #          - Switch # Build a Nintendo Switch player
-    # steps
+        cloudRunnerCluster:
+          - aws
+          - local-docker
+          - k8s
     steps:
       - name: Checkout (default)
         uses: actions/checkout@v2
-        if: github.event.event_type != 'pull_request_target'
         with:
           lfs: true
+      - uses: google-github-actions/setup-gcloud@v0
+        with:
+          version: '288.0.0'
+          service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: Get GKE cluster credentials
+        run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -66,81 +57,117 @@ jobs:
           aws-region: eu-west-2
       - run: yarn
       - run: yarn run cli --help
-      - run: yarn run test "caching"
-      - run: yarn run test-i-aws
+      - run: yarn run test "cloud-runner-run-twice-retaining" --detectOpenHandles --forceExit --runInBand
+        if: matrix.CloudRunnerCluster == 'aws' || matrix.CloudRunnerCluster == 'k8s'
+        timeout-minutes: 180
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
-          PROJECT_PATH: ${{ matrix.projectPath }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          TARGET_PLATFORM: ${{ matrix.targetPlatform }}
+          PROJECT_PATH: test-project
+          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
-      - uses: ./
-        id: aws-fargate-unity-build
-        timeout-minutes: 25
-        with:
-          cloudRunnerCluster: aws
+          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+      - run: yarn run test-i --detectOpenHandles --forceExit --runInBand
+        if: matrix.CloudRunnerCluster == 'local-docker'
+        timeout-minutes: 180
+        env:
+          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
+          PROJECT_PATH: test-project
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_PLATFORM: StandaloneWindows64
+          cloudRunnerTests: true
           versioning: None
-          projectPath: ${{ matrix.projectPath }}
-          unityVersion: ${{ matrix.unityVersion }}
-          targetPlatform: ${{ matrix.targetPlatform }}
-          githubToken: ${{ secrets.GITHUB_TOKEN }}
-          postBuildSteps: |
-            - name: upload
-              image: amazon/aws-cli
-              commands: |
-                aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
-                aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
-                aws configure set region $AWS_DEFAULT_REGION --profile default
-                aws s3 ls
-                aws s3 ls game-ci-test-storage
-                ls /data/cache/$CACHE_KEY
-                ls /data/cache/$CACHE_KEY/build
-                aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.tar
-              secrets:
-              - name: awsAccessKeyId
-                value: ${{ secrets.AWS_ACCESS_KEY_ID }}
-              - name: awsSecretAccessKey
-                value: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-              - name: awsDefaultRegion
-                value: eu-west-2
-      - run: |
-          aws s3 cp s3://game-ci-test-storage/${{ steps.aws-fargate-unity-build.outputs.CACHE_KEY }}/build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar
-          ls
-      - run: yarn run cli -m aws-garbage-collect
-      ###########################
-      #          Upload         #
-      ###########################
-      # download from cloud storage
-      - uses: actions/upload-artifact@v2
-        with:
-          name: AWS Build (${{ matrix.targetPlatform }})
-          path: build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar
-          retention-days: 14
-  k8sBuilds:
-    name: K8s (GKE Autopilot) build for ${{ matrix.targetPlatform }} on version ${{ matrix.unityVersion }}
+          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+
+  buildTargetTests:
+    name: Build Tests - Targets
+    if: github.event.event_type != 'pull_request_target'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        unityVersion:
-          #  - 2019.2.11f1
-          - 2019.3.15f1
+        cloudRunnerCluster:
+          #- aws
+          - local-docker
+          #- k8s
         targetPlatform:
-          #  - StandaloneWindows64
-          - StandaloneLinux64
+          - StandaloneOSX # Build a macOS standalone (Intel 64-bit).
+          # - StandaloneWindows64 # Build a Windows 64-bit standalone.
+          - StandaloneLinux64 # Build a Linux 64-bit standalone.
+          - WebGL # WebGL.
+          - iOS # Build an iOS player.
+          - Android # Build an Android .apk.
     steps:
-      ###########################
-      #         Checkout        #
-      ###########################
-      - uses: actions/checkout@v2
-        if: github.event.event_type != 'pull_request_target'
+      - name: Checkout (default)
+        uses: actions/checkout@v2
+        with:
+          lfs: true
+
+      - uses: google-github-actions/setup-gcloud@v0
+        with:
+          version: '288.0.0'
+          service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: Get GKE cluster credentials
+        run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+      - run: yarn
+      - uses: ./
+        id: unity-build
+        timeout-minutes: 90
+        env:
+          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
+        with:
+          cloudRunnerTests: true
+          versioning: None
+          projectPath: test-project
+          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
+          targetPlatform: ${{ matrix.targetPlatform }}
+          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          customStepFiles: aws-s3-upload-build,aws-s3-pull-cache,aws-s3-upload-cache
+      - run: |
+          aws s3 cp s3://game-ci-test-storage/cloud-runner-cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
+          ls
+      - run: yarn run cli -m list-resources
+        env:
+          cloudRunnerTests: true
+          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
+          path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
+          retention-days: 14
+  buildTests:
+    name: Build Tests - Providers
+    if: github.event.event_type != 'pull_request_target'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        cloudRunnerCluster:
+          - aws
+          - local-docker
+          - k8s
+        targetPlatform:
+          #- StandaloneOSX # Build a macOS standalone (Intel 64-bit).
+          - StandaloneWindows64 # Build a Windows 64-bit standalone.
+          #- StandaloneLinux64 # Build a Linux 64-bit standalone.
+          #- WebGL # WebGL.
+          #- iOS # Build an iOS player.
+          #- Android # Build an Android .apk.
+    # steps
+    steps:
+      - name: Checkout (default)
+        uses: actions/checkout@v2
         with:
           lfs: true
 
-      ###########################
-      #          Setup          #
-      ###########################
       - uses: google-github-actions/setup-gcloud@v0
         with:
           version: '288.0.0'
@@ -149,70 +176,36 @@ jobs:
       - name: Get GKE cluster credentials
         run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
 
-      ###########################
-      # Cloud Runner Test Suite #
-      ###########################
-      - uses: actions/setup-node@v2
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
         with:
-          node-version: 12.x
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
       - run: yarn
-      - run: yarn run cli --help
-      - run: yarn run test "caching"
-      - name: Cloud Runner Test Suite
-        run: yarn run test-i-k8s --detectOpenHandles --forceExit
+      - uses: ./
+        id: unity-build
+        timeout-minutes: 90
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
-          PROJECT_PATH: ${{ matrix.projectPath }}
-          TARGET_PLATFORM: ${{ matrix.targetPlatform }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          KUBE_CONFIG: ${{ steps.read-base64.outputs.base64 }}
-          unityVersion: ${{ matrix.unityVersion }}
+        with:
           cloudRunnerTests: true
           versioning: None
-
-      ###########################
-      # Cloud Runner Build Test #
-      ###########################
-      - name: Cloud Runner Build Test
-        uses: ./
-        id: k8s-unity-build
-        timeout-minutes: 30
-        with:
-          cloudRunnerCluster: k8s
-          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
-          targetPlatform: ${{ matrix.targetPlatform }}
-          kubeConfig: ${{ steps.read-base64.outputs.base64 }}
-          githubToken: ${{ secrets.GITHUB_TOKEN }}
           projectPath: test-project
-          unityVersion: ${{ matrix.unityVersion }}
-          versioning: None
-          postBuildSteps: |
-            - name: upload
-              image: amazon/aws-cli
-              commands: |
-                aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
-                aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
-                aws configure set region $AWS_DEFAULT_REGION --profile default
-                aws s3 ls
-                aws s3 ls game-ci-test-storage
-                ls /data/cache/$CACHE_KEY
-                aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.tar
-              secrets:
-              - name: awsAccessKeyId
-                value: ${{ secrets.AWS_ACCESS_KEY_ID }}
-              - name: awsSecretAccessKey
-                value: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-              - name: awsDefaultRegion
-                value: eu-west-2
+          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
+          targetPlatform: ${{ matrix.targetPlatform }}
+          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          customStepFiles: aws-s3-upload-build,aws-s3-pull-cache,aws-s3-upload-cache
       - run: |
-          aws s3 cp s3://game-ci-test-storage/${{ steps.k8s-unity-build.outputs.CACHE_KEY }}/build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar
+          aws s3 cp s3://game-ci-test-storage/cloud-runner-cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
           ls
-      ###########################
-      #          Upload         #
-      ###########################
-      # download from cloud storage
+      - run: yarn run cli -m list-resources
+        if: always()
+        env:
+          cloudRunnerTests: true
+          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
       - uses: actions/upload-artifact@v2
         with:
-          name: K8s Build (${{ matrix.targetPlatform }})
-          path: build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar
+          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
+          path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
           retention-days: 14