Cloud runner v0.2 - continued quality of life improvements (#387)

* Update cloud-runner-aws-pipeline.yml

* Update cloud-runner-k8s-pipeline.yml

* yarn build

* yarn build

* correct branch ref

* correct branch ref passed to target repo

* Create k8s-tests.yml

* Delete k8s-tests.yml

* correct branch ref passed to target repo

* correct branch ref passed to target repo

* Always describe AWS tasks for now, because unstable error handling

* Remove unused tree commands

* Use lfs guid sum

* Simple override cache push

* Simple override cache push and pull override to allow pure cloud storage driven caching

* Removal of early branch (breaks lfs caching)

* Remove unused tree commands

* Update action.yml

* Update action.yml

* Support cache and input override commands as input + full support custom hooks

* Increase k8s timeout

* replace filename being appended for unknclear reason

* cache key should not contain whitespaces

* Always try and deploy rook for k8s

* Apply k8s files for rook

* Update action.yml

* Apply k8s files for rook

* Apply k8s files for rook

* cache test and action description for kuber storage class

* Correct test and implement dependency health check and start

* GCP-secret run, cache key

* lfs smudge set explicit and undo explicit

* Run using external secret provider to speed up input

* Update cloud-runner-aws-pipeline.yml

* Add nodejs as build step dependency

* Add nodejs as build step dependency

* Cloud Runner Tests must be specified to capture logs from cloud runner for tests

* Cloud Runner Tests must be specified to capture logs from cloud runner for tests

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs

* better defaults for new inputs

* better defaults

* merge latest

* force build update

* use npm n to update node in unity builder

* use npm n to update node in unity builder

* use npm n to update node in unity builder

* correct new line

* quiet zipping

* quiet zipping

* default secrets for unity username and password

* default secrets for unity username and password

* ls active directory before lfs install

* Get cloud runner secrets from

* Get cloud runner secrets from

* Cleanup setup of default secrets

* Various fixes

* Cleanup setup of default secrets

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* Various fixes

* AWS secrets manager support

* less caching logs

* default k8s storage class to pd-standard

* more readable build commands

* Capture aws exit code 1 reliably

* Always replace /head from branch

* k8s default storage class to standard-rwo

* cleanup

* further cleanup input

* further cleanup input

* further cleanup input

* further cleanup input

* further cleanup input

* folder sizes to inspect caching

* dir command for local cloud runner test

* k8s wait for pending because pvc will not create earlier

* prefer k8s standard storage

* handle empty string as cloud runner cluster input

* local-system is now used for cloud runner test implementation AND correctly unset test CLI input

* local-system is now used for cloud runner test implementation AND correctly unset test CLI input

* fix unterminated quote

* fix unterminated quote

* do not share build parameters in tests - in cloud runner this will cause conflicts with resouces of the same name

* remove head and heads from branch prefix

* fix reversed caching direction of cache-push

* fixes

* fixes

* fixes

* cachePull cli

* fixes

* fixes

* fixes

* fixes

* fixes

* order cache test to be first

* order cache test to be first

* fixes

* populate cache key instead of using branch

* cleanup cli

* garbage-collect-aws cli can iterate over aws resources and cli scans all ts files

* import cli methods

* import cli files explicitly

* import cli files explicitly

* import cli files explicitly

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* import cli methods

* log parameters in cloud runner parameter test

* log parameters in cloud runner parameter test

* log parameters in cloud runner parameter test

* Cloud runner param test before caching because we have a fast local cache test now

* Using custom build path relative to repo root rather than project root

* aws-garbage-collect at end of pipeline

* aws-garbage-collect do not actually delete anything for now - just list

* remove some legacy du commands

* Update cloud-runner-aws-pipeline.yml

* log contents after cache pull and fix some scenarios with duplicate secrets

* log contents after cache pull and fix some scenarios with duplicate secrets

* log contents after cache pull and fix some scenarios with duplicate secrets

* PR comments

* Replace guid with uuid package

* use fileExists lambda instead of stat to check file exists in caching

* build failed results in core error message

* Delete sample.txt

* cloud-runner-system prefix changed to cloud-runner

* Update cloud-runner-aws-pipeline.yml

* remove du from caching, should run manually if interested in size, adds too much runtime to job to include by default

* github ephemeral pipeline support

* github ephemeral pipeline support

* Merge remote-tracking branch 'origin/main' into cloud-runner-develop

# Conflicts:
#	dist/index.js.map
#	src/model/cloud-runner/providers/aws/aws-task-runner.ts
#	src/model/cloud-runner/providers/aws/index.ts

* garbage collection

* garbage collection

* self hosted runner pipeline

* self hosted runner pipeline

* self hosted runner pipeline

* self hosted runner pipeline

* self hosted runner pipeline

* self hosted runner pipeline

* self hosted runner pipeline

* self hosted runner pipeline

* self hosted runner pipeline

* self hosted runner pipeline

* ephemeral runner pipeline

* ephemeral runner pipeline

* ephemeral runner pipeline

* download runner each time

* download runner each time

* download runner each time

* garbage collect all older than 1d as part of cleanup

* download runner each time

* number container cpu and memory for aws

* per provider container defaults

* per provider container defaults

* per provider container defaults

* per provider container defaults

* Skip printing size unless cloudRunnerIntegrationTests is true

* transition zip usage in cache to uncompressed tar for speed

* transition zip usage in cache to uncompressed tar for speed

* transition zip usage in cache to uncompressed tar for speed

* transition zip usage in cache to uncompressed tar for speed

* per provider container defaults

* per provider container defaults

* per provider container defaults

* per provider container defaults

* per provider container defaults

* per provider container defaults

* per provider container defaults

* per provider container defaults

* baked in cloud formation template

* baked in cloud formation template

* baked in cloud formation template

* baked in cloud formation template

* baked in cloud formation template

* baked in cloud formation template

* baked in cloud formation template

* baked in cloud formation template

* better aws commands

* better aws commands

* parse number for cloud formation template

* remove container resource defaults from actions yaml

* remove container resource defaults from actions yaml

* skip all input readers when cloud runner is local

* prefer fs/promises

* actually set aws cloud runner step as failure if unity build fails

* default to 3gb of ram - webgl fails on 2

src/model/cloud-runner/remote-client/caching.ts

@@ -70,17 +70,17 @@ export class Caching {
           return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
         });
       };
-      await CloudRunnerSystem.Run(`zip -q -r ${cacheArtifactName}.zip ${path.basename(sourceFolder)}`);
-      assert(await fileExists(`${cacheArtifactName}.zip`), 'cache zip exists');
+      await CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar ${path.basename(sourceFolder)}`);
+      assert(await fileExists(`${cacheArtifactName}.tar`), 'cache archive exists');
       assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
       if (CloudRunner.buildParameters.cachePushOverrideCommand) {
         await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePushOverrideCommand));
       }
-      await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.zip ${cacheFolder}`);
-      RemoteClientLogger.log(`moved ${cacheArtifactName}.zip to ${cacheFolder}`);
+      await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar ${cacheFolder}`);
+      RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
       assert(
-        await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.zip`),
-        'cache zip exists inside cache folder',
+        await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar`),
+        'cache archive exists inside cache folder',
       );
     } catch (error) {
       process.chdir(`${startPath}`);
@@ -101,14 +101,14 @@ export class Caching {
         await fs.promises.mkdir(destinationFolder);
       }
 
-      const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .zip$ | head -1`))
+      const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar$ | head -1`))
         .replace(/\n/g, ``)
-        .replace('.zip', '');
+        .replace('.tar', '');
 
       process.chdir(cacheFolder);
 
       const cacheSelection =
-        cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.zip`)) ? cacheArtifactName : latestInBranch;
+        cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar`)) ? cacheArtifactName : latestInBranch;
       await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
 
       // eslint-disable-next-line func-style
@@ -127,12 +127,12 @@ export class Caching {
         await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePullOverrideCommand));
       }
 
-      if (await fileExists(`${cacheSelection}.zip`)) {
+      if (await fileExists(`${cacheSelection}.tar`)) {
         const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
         await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
-        RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.zip`);
+        RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar`);
         const fullResultsFolder = path.join(cacheFolder, resultsFolder);
-        await CloudRunnerSystem.Run(`unzip -q ${cacheSelection}.zip -d ${path.basename(resultsFolder)}`);
+        await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar -C ${fullResultsFolder}`);
         RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
         assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
         const destinationParentFolder = path.resolve(destinationFolder, '..');
@@ -143,7 +143,6 @@ export class Caching {
         await CloudRunnerSystem.Run(
           `mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
         );
-        await CloudRunnerSystem.Run(`du -sh ${path.join(destinationParentFolder, path.basename(destinationFolder))}`);
         const contents = await fs.promises.readdir(
           path.join(destinationParentFolder, path.basename(destinationFolder)),
         );
@@ -153,7 +152,7 @@ export class Caching {
       } else {
         RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
         if (cacheSelection !== ``) {
-          RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.zip doesn't exist ${destinationFolder}`);
+          RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar doesn't exist ${destinationFolder}`);
           throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
         }
       }

src/model/cloud-runner/remote-client/index.ts

@@ -45,8 +45,10 @@ export class RemoteClient {
   }
 
   private static async sizeOfFolder(message: string, folder: string) {
-    CloudRunnerLogger.log(`Size of ${message}`);
-    await CloudRunnerSystem.Run(`du -sh ${folder}`);
+    if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
+      CloudRunnerLogger.log(`Size of ${message}`);
+      await CloudRunnerSystem.Run(`du -sh ${folder}`);
+    }
   }
 
   private static async cloneRepoWithoutLFSFiles() {