Cloud runner develop - latest fixes (#524)

2023-03-27 12:14:23 +01:00
parent 309d668d63
commit 7abb3a409d
91 changed files with 4943 additions and 3599 deletions
--- a/.github/workflows/build-tests-ubuntu.yml
+++ b/.github/workflows/build-tests-ubuntu.yml
@@ -49,7 +49,7 @@ jobs:
        exclude:
          - targetPlatform: Android
            unityVersion: 2022.2.7f1
-        cloudRunnerCluster:
+        providerStrategy:
          # - local-docker
          - local
        projectPath:
@@ -109,7 +109,7 @@ jobs:
          unityVersion: ${{ matrix.unityVersion }}
          targetPlatform: ${{ matrix.targetPlatform }}
          customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          providerStrategy: ${{ matrix.providerStrategy }}

      ###########################
      #          Upload         #
--- a/.github/workflows/cloud-runner-async-checks.yml
+++ b/.github/workflows/cloud-runner-async-checks.yml
@@ -23,7 +23,7 @@ env:
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-github-pipelines
+  AWS_STACK_NAME: game-ci-github-pipelines
  CLOUD_RUNNER_BRANCH: ${{ github.ref }}
  CLOUD_RUNNER_DEBUG: true
  CLOUD_RUNNER_DEBUG_TREE: true
@@ -39,20 +39,21 @@ jobs:
    if: github.event.event_type != 'pull_request_target'
    runs-on: ubuntu-latest
    steps:
-      - name: Checkout (default)
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-      - run: yarn
-      - run: yarn run cli -m checks-update
-        timeout-minutes: 180
+      - timeout-minutes: 180
        env:
          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
          PROJECT_PATH: test-project
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TARGET_PLATFORM: StandaloneWindows64
          cloudRunnerTests: true
          versioning: None
          CLOUD_RUNNER_CLUSTER: local-docker
-          AWS_BASE_STACK_NAME: game-ci-github-pipelines
+          AWS_STACK_NAME: game-ci-github-pipelines
          CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
+        run: |
+          git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
+          cd unity-builder
+          yarn
+          ls
+          yarn run cli -m checks-update
--- a/.github/workflows/cloud-runner-ci-pipeline.yml
+++ b/.github/workflows/cloud-runner-ci-pipeline.yml
@@ -21,81 +21,140 @@ env:
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-team-pipelines
+  AWS_STACK_NAME: game-ci-team-pipelines
  CLOUD_RUNNER_BRANCH: ${{ github.ref }}
-  CLOUD_RUNNER_DEBUG: true
-  CLOUD_RUNNER_DEBUG_TREE: true
  DEBUG: true
  UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
  PROJECT_PATH: test-project
  UNITY_VERSION: 2019.3.15f1
  USE_IL2CPP: false
  USE_GKE_GCLOUD_AUTH_PLUGIN: true
+  GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

 jobs:
-  integrationTests:
-    name: Integration Tests
+  smokeTests:
+    name: Smoke Tests
    if: github.event.event_type != 'pull_request_target'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
-        cloudRunnerCluster:
-          - aws
+        test:
+          #- 'cloud-runner-async-workflow'
+          - 'cloud-runner-caching'
+          # - 'cloud-runner-end2end-caching'
+          # - 'cloud-runner-end2end-retaining'
+          - 'cloud-runner-environment'
+          - 'cloud-runner-hooks'
+          - 'cloud-runner-local-persistence'
+          - 'cloud-runner-locking-core'
+          - 'cloud-runner-locking-get-locked'
+        providerStrategy:
+          #- aws
          - local-docker
-          - k8s
+          #- k8s
    steps:
      - name: Checkout (default)
        uses: actions/checkout@v3
        with:
          lfs: false
-      - uses: google-github-actions/auth@v1
-        with:
-          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v1'
-      - name: Get GKE cluster credentials
-        run: |
-          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
-          gcloud components install gke-gcloud-auth-plugin
-          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
+      - uses: google-github-actions/auth@v1
+        if: matrix.providerStrategy == 'k8s'
+        with:
+          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: 'Set up Cloud SDK'
+        if: matrix.providerStrategy == 'k8s'
+        uses: 'google-github-actions/setup-gcloud@v1.1.0'
+      - name: Get GKE cluster credentials
+        if: matrix.providerStrategy == 'k8s'
+        run: |
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud components install gke-gcloud-auth-plugin
+          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
      - run: yarn
-      - run: yarn run test "cloud-runner-async-workflow" --detectOpenHandles --forceExit --runInBand
-        if: matrix.CloudRunnerCluster != 'local-docker'
-        timeout-minutes: 180
+      - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
+        timeout-minutes: 35
        env:
          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
          PROJECT_PATH: test-project
-          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TARGET_PLATFORM: StandaloneWindows64
          cloudRunnerTests: true
          versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: yarn run test-i --detectOpenHandles --forceExit --runInBand
-        if: matrix.CloudRunnerCluster == 'local-docker'
-        timeout-minutes: 180
+          CLOUD_RUNNER_CLUSTER: ${{ matrix.providerStrategy }}
+  tests:
+    # needs:
+    #   - smokeTests
+    #   - buildTargetTests
+    name: Integration Tests
+    if: github.event.event_type != 'pull_request_target'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        providerStrategy:
+          - aws
+          - local-docker
+          - k8s
+        test:
+          - 'cloud-runner-async-workflow'
+          #- 'cloud-runner-caching'
+          - 'cloud-runner-end2end-locking'
+          - 'cloud-runner-end2end-caching'
+          - 'cloud-runner-end2end-retaining'
+          - 'cloud-runner-environment'
+          #- 'cloud-runner-hooks'
+          - 'cloud-runner-s3-steps'
+          #- 'cloud-runner-local-persistence'
+          #- 'cloud-runner-locking-core'
+          #- 'cloud-runner-locking-get-locked'
+    steps:
+      - name: Checkout (default)
+        uses: actions/checkout@v2
+        with:
+          lfs: false
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+      - uses: google-github-actions/auth@v1
+        if: matrix.providerStrategy == 'k8s'
+        with:
+          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: 'Set up Cloud SDK'
+        if: matrix.providerStrategy == 'k8s'
+        uses: 'google-github-actions/setup-gcloud@v1.1.0'
+      - name: Get GKE cluster credentials
+        if: matrix.providerStrategy == 'k8s'
+        run: |
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud components install gke-gcloud-auth-plugin
+          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
+      - run: yarn
+      - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
+        timeout-minutes: 60
        env:
          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
          PROJECT_PATH: test-project
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TARGET_PLATFORM: StandaloneWindows64
          cloudRunnerTests: true
          versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
-  localBuildTests:
+          PROVIDER_STRATEGY: ${{ matrix.providerStrategy }}
+  buildTargetTests:
    name: Local Build Target Tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
-        cloudRunnerCluster:
+        providerStrategy:
          #- aws
          - local-docker
          #- k8s
@@ -114,20 +173,18 @@ jobs:
      - run: yarn
      - uses: ./
        id: unity-build
-        timeout-minutes: 90
+        timeout-minutes: 30
        env:
          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
        with:
          cloudRunnerTests: true
          versioning: None
-          projectPath: test-project
-          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
          targetPlatform: ${{ matrix.targetPlatform }}
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          providerStrategy: ${{ matrix.providerStrategy }}
      - run: |
          cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
      - uses: actions/upload-artifact@v3
        with:
-          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
+          name: ${{ matrix.providerStrategy }} Build (${{ matrix.targetPlatform }})
          path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
          retention-days: 14