Cloud runner develop - better parameterization of s3 usage, improved async workflow and GC, github checks early integration (#479)

* custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * add 3 new premade steps, steam-deploy-client, steam-deploy-project, aws-s3-pull-build * fix * fix * fix * continue building async-workflow support * test checks * test checks * test checks * move github checks within build workflow * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * cleanup logging * disable lz4 compression by default * disable lz4 compression by default * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * disable lz4 compression by default * disable lz4 compression by default * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * workflow * workflow * workflow * workflow * workflow * workflow * workflow * workflow * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3
2023-01-20 17:40:57 +00:00
parent a45155c578
commit e334dc785a
39 changed files with 3083 additions and 399 deletions
--- a/.github/workflows/cloud-runner-async-checks.yml
+++ b/.github/workflows/cloud-runner-async-checks.yml
@@ -8,7 +8,7 @@ on:
        required: false
        default: ''

-permissions: 
+permissions:
  checks: write

 env:
@@ -44,7 +44,6 @@ jobs:
        with:
          lfs: false
      - run: yarn
-      - run: yarn run cli --help
      - run: yarn run cli -m checks-update
        timeout-minutes: 180
        env:
--- a/.github/workflows/cloud-runner-local-pipeline.yml
+++ b/.github/workflows/cloud-runner-local-pipeline.yml
@@ -1,95 +0,0 @@
-name: Cloud Runner Local
-
-on:
-  push: { branches: ['!cloud-runner-develop', '!cloud-runner-preview', '!main'] }
-#  push: { branches: [main] }
-#  pull_request:
-#    paths-ignore:
-#      - '.github/**'
-
-jobs:
-  integrationTests:
-    name: Integration Tests
-    if: github.event.event_type != 'pull_request_target'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        cloudRunnerCluster:
-          - local-docker
-        targetPlatform:
-          - StandaloneWindows64 # Build a Windows 64-bit standalone.
-    # steps
-    steps:
-      - name: Checkout (default)
-        uses: actions/checkout@v2
-        with:
-          lfs: true
-      - run: yarn
-      - run: yarn run cli --help
-      - run: yarn run test-i --detectOpenHandles --forceExit --runInBand
-        env:
-          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
-          PROJECT_PATH: ${{ matrix.projectPath }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          TARGET_PLATFORM: ${{ matrix.targetPlatform }}
-          cloudRunnerTests: true
-          versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
-  buildTests:
-    name: Build Tests
-    if: github.event.event_type != 'pull_request_target'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        cloudRunnerCluster:
-          - local-docker
-        targetPlatform:
-          - StandaloneOSX # Build a macOS standalone (Intel 64-bit).
-          - StandaloneWindows64 # Build a Windows 64-bit standalone.
-          - StandaloneLinux64 # Build a Linux 64-bit standalone.
-          - WebGL # WebGL.
-          - iOS # Build an iOS player.
-          - Android # Build an Android .apk.
-          #          - StandaloneWindows # Build a Windows standalone.
-          #          - WSAPlayer # Build an Windows Store Apps player.
-          #          - PS4 # Build a PS4 Standalone.
-          #          - XboxOne # Build a Xbox One Standalone.
-          #          - tvOS # Build to Apple's tvOS platform.
-          #          - Switch # Build a Nintendo Switch player
-    # steps
-    steps:
-      - name: Checkout (default)
-        uses: actions/checkout@v2
-        with:
-          lfs: true
-      - uses: ./
-        id: unity-build
-        timeout-minutes: 25
-        env:
-          CLOUD_RUNNER_BRANCH: ${{ github.ref }}
-          CLOUD_RUNNER_DEBUG: true
-          CLOUD_RUNNER_DEBUG_TREE: true
-          DEBUG: true
-          PROJECT_PATH: test-project
-          UNITY_VERSION: 2019.3.15f1
-          USE_IL2CPP: false
-        with:
-          cloudRunnerTests: true
-          versioning: None
-          projectPath: ${{ matrix.projectPath }}
-          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
-          targetPlatform: ${{ matrix.targetPlatform }}
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
-      - run: |
-          mv ./cloud-runner-cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
-          ls
-      ###########################
-      #          Upload         #
-      ###########################
-      - uses: actions/upload-artifact@v2
-        with:
-          name: Local Build (${{ matrix.targetPlatform }})
-          path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
-          retention-days: 14
--- a/.github/workflows/cloud-runner-pipeline.yml
+++ b/.github/workflows/cloud-runner-pipeline.yml
@@ -2,6 +2,12 @@ name: Cloud Runner CI Pipeline

 on:
  push: { branches: [cloud-runner-develop, cloud-runner-preview, main] }
+  workflow_dispatch:
+
+permissions:
+  checks: write
+  contents: read
+  actions: write

 env:
  GKE_ZONE: 'us-central1'
@@ -15,7 +21,7 @@ env:
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-github-pipelines
+  AWS_BASE_STACK_NAME: game-ci-team-pipelines
  CLOUD_RUNNER_BRANCH: ${{ github.ref }}
  CLOUD_RUNNER_DEBUG: true
  CLOUD_RUNNER_DEBUG_TREE: true
@@ -24,6 +30,7 @@ env:
  PROJECT_PATH: test-project
  UNITY_VERSION: 2019.3.15f1
  USE_IL2CPP: false
+  USE_GKE_GCLOUD_AUTH_PLUGIN: true

 jobs:
  integrationTests:
@@ -41,14 +48,17 @@ jobs:
      - name: Checkout (default)
        uses: actions/checkout@v2
        with:
-          lfs: true
-      - uses: google-github-actions/setup-gcloud@v0
+          lfs: false
+      - uses: google-github-actions/auth@v1
        with:
-          version: '288.0.0'
-          service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
-          service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v1'
      - name: Get GKE cluster credentials
-        run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
+        run: |
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud components install gke-gcloud-auth-plugin
+          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
@@ -56,9 +66,8 @@ jobs:
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
      - run: yarn
-      - run: yarn run cli --help
-      - run: yarn run test "cloud-runner-run-twice-retaining" --detectOpenHandles --forceExit --runInBand
-        if: matrix.CloudRunnerCluster == 'aws' || matrix.CloudRunnerCluster == 'k8s'
+      - run: yarn run test "cloud-runner-async-workflow" --detectOpenHandles --forceExit --runInBand
+        if: matrix.CloudRunnerCluster != 'local-docker'
        timeout-minutes: 180
        env:
          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
@@ -68,6 +77,7 @@ jobs:
          cloudRunnerTests: true
          versioning: None
          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - run: yarn run test-i --detectOpenHandles --forceExit --runInBand
        if: matrix.CloudRunnerCluster == 'local-docker'
        timeout-minutes: 180
@@ -79,10 +89,8 @@ jobs:
          cloudRunnerTests: true
          versioning: None
          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
-
-  buildTargetTests:
-    name: Build Tests - Targets
-    if: github.event.event_type != 'pull_request_target'
+  localBuildTests:
+    name: Local Build Target Tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
@@ -93,7 +101,7 @@ jobs:
          #- k8s
        targetPlatform:
          - StandaloneOSX # Build a macOS standalone (Intel 64-bit).
-          # - StandaloneWindows64 # Build a Windows 64-bit standalone.
+          - StandaloneWindows64 # Build a Windows 64-bit standalone.
          - StandaloneLinux64 # Build a Linux 64-bit standalone.
          - WebGL # WebGL.
          - iOS # Build an iOS player.
@@ -102,21 +110,7 @@ jobs:
      - name: Checkout (default)
        uses: actions/checkout@v2
        with:
-          lfs: true
-
-      - uses: google-github-actions/setup-gcloud@v0
-        with:
-          version: '288.0.0'
-          service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
-          service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
-      - name: Get GKE cluster credentials
-        run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-2
+          lfs: false
      - run: yarn
      - uses: ./
        id: unity-build
@@ -130,82 +124,10 @@ jobs:
          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
          targetPlatform: ${{ matrix.targetPlatform }}
          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
-          customStepFiles: aws-s3-upload-build,aws-s3-pull-cache,aws-s3-upload-cache
      - run: |
-          aws s3 cp s3://game-ci-test-storage/cloud-runner-cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
-          ls
-      - run: yarn run cli -m list-resources
-        env:
-          cloudRunnerTests: true
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+          cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
      - uses: actions/upload-artifact@v2
        with:
          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
-          path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
-          retention-days: 14
-  buildTests:
-    name: Build Tests - Providers
-    if: github.event.event_type != 'pull_request_target'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        cloudRunnerCluster:
-          - aws
-          - local-docker
-          - k8s
-        targetPlatform:
-          #- StandaloneOSX # Build a macOS standalone (Intel 64-bit).
-          - StandaloneWindows64 # Build a Windows 64-bit standalone.
-          #- StandaloneLinux64 # Build a Linux 64-bit standalone.
-          #- WebGL # WebGL.
-          #- iOS # Build an iOS player.
-          #- Android # Build an Android .apk.
-    # steps
-    steps:
-      - name: Checkout (default)
-        uses: actions/checkout@v2
-        with:
-          lfs: true
-
-      - uses: google-github-actions/setup-gcloud@v0
-        with:
-          version: '288.0.0'
-          service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
-          service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
-      - name: Get GKE cluster credentials
-        run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-2
-      - run: yarn
-      - uses: ./
-        id: unity-build
-        timeout-minutes: 90
-        env:
-          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
-        with:
-          cloudRunnerTests: true
-          versioning: None
-          projectPath: test-project
-          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
-          targetPlatform: ${{ matrix.targetPlatform }}
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
-          customStepFiles: aws-s3-upload-build,aws-s3-pull-cache,aws-s3-upload-cache
-      - run: |
-          aws s3 cp s3://game-ci-test-storage/cloud-runner-cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
-          ls
-      - run: yarn run cli -m list-resources
-        if: always()
-        env:
-          cloudRunnerTests: true
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
-      - uses: actions/upload-artifact@v2
-        with:
-          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
-          path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
+          path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
          retention-days: 14