Cloud Runner v2 (#310)

src/model/cli/cli-decorator.ts

@@ -0,0 +1,23 @@
+const targets = new Array();
+export function CliFunction(key: string, description: string) {
+  return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
+    targets.push({
+      target,
+      propertyKey,
+      descriptor,
+      key,
+      description,
+    });
+  };
+}
+export function GetCliFunctions(key) {
+  return targets.find((x) => x.key === key);
+}
+export function GetAllCliModes() {
+  return targets.map((x) => {
+    return {
+      key: x.key,
+      description: x.description,
+    };
+  });
+}

src/model/cli/cli.ts

@@ -0,0 +1,88 @@
+import { Command } from 'commander-ts';
+import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
+import * as core from '@actions/core';
+import { ActionYamlReader } from '../input-readers/action-yaml';
+import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
+import { CliFunction, GetAllCliModes, GetCliFunctions } from './cli-decorator';
+import { RemoteClientLogger } from './remote-client/remote-client-services/remote-client-logger';
+import { CloudRunnerState } from '../cloud-runner/state/cloud-runner-state';
+import { SetupCloudRunnerRepository } from './remote-client/setup-cloud-runner-repository';
+import * as SDK from 'aws-sdk';
+
+export class CLI {
+  static async RunCli(options: any): Promise<void> {
+    Input.githubInputEnabled = false;
+
+    const results = GetCliFunctions(options.mode);
+
+    if (results === undefined || results.length === 0) {
+      throw new Error('no CLI mode found');
+    }
+
+    CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
+
+    options.versioning = 'None';
+    Input.cliOptions = options;
+    return await results.target[results.propertyKey]();
+  }
+  static isCliMode(options: any) {
+    return options.mode !== undefined && options.mode !== '';
+  }
+
+  public static SetupCli() {
+    const program = new Command();
+    program.version('0.0.1');
+    const properties = Object.getOwnPropertyNames(Input);
+    core.info(`\n`);
+    core.info(`INPUT:`);
+    const actionYamlReader: ActionYamlReader = new ActionYamlReader();
+    for (const element of properties) {
+      program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
+      if (Input[element] !== undefined && Input[element] !== '' && typeof Input[element] !== `function`) {
+        core.info(`${element} ${Input[element]}`);
+      }
+    }
+    core.info(`\n`);
+    program.option(
+      '-m, --mode <mode>',
+      GetAllCliModes()
+        .map((x) => `${x.key} (${x.description})`)
+        .join(` | `),
+    );
+    program.parse(process.argv);
+
+    return program.opts();
+  }
+
+  @CliFunction(`cli`, `runs a cloud runner build`)
+  public static async CLIBuild(): Promise<string> {
+    const buildParameter = await BuildParameters.create();
+    const baseImage = new ImageTag(buildParameter);
+    return await CloudRunner.run(buildParameter, baseImage.toString());
+  }
+
+  @CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`)
+  static async runRemoteClientJob() {
+    const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
+    RemoteClientLogger.log(`Build Params:
+      ${JSON.stringify(buildParameter, undefined, 4)}
+    `);
+    CloudRunnerState.setup(buildParameter);
+    await SetupCloudRunnerRepository.run();
+  }
+
+  @CliFunction(`cach-push`, `push to cache`)
+  static async cachePush() {}
+
+  @CliFunction(`cach-pull`, `pull from cache`)
+  static async cachePull() {}
+
+  @CliFunction(`garbage-collect-aws`, `garbage collect aws`)
+  static async garbageCollectAws() {
+    process.env.AWS_REGION = Input.region;
+    const CF = new SDK.CloudFormation();
+
+    const stacks = await CF.listStacks().promise();
+    CloudRunnerLogger.log(JSON.stringify(stacks, undefined, 4));
+  }
+}

src/model/cli/remote-client/remote-client-services/caching.ts

@@ -0,0 +1,117 @@
+import { assert } from 'console';
+import fs from 'fs';
+import path from 'path';
+import { Input } from '../../..';
+import CloudRunnerLogger from '../../../cloud-runner/services/cloud-runner-logger';
+import { CloudRunnerState } from '../../../cloud-runner/state/cloud-runner-state';
+import { CloudRunnerSystem } from './cloud-runner-system';
+import { LFSHashing } from './lfs-hashing';
+import { RemoteClientLogger } from './remote-client-logger';
+
+export class Caching {
+  public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheKey: string) {
+    const startPath = process.cwd();
+    try {
+      if (!fs.existsSync(cacheFolder)) {
+        await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
+      }
+      process.chdir(path.resolve(sourceFolder, '..'));
+
+      if (Input.cloudRunnerTests) {
+        CloudRunnerLogger.log(
+          `Hashed cache folder ${await LFSHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
+            sourceFolder,
+          )}`,
+        );
+      }
+
+      if (Input.cloudRunnerTests) {
+        await CloudRunnerSystem.Run(`ls ${path.basename(sourceFolder)}`);
+      }
+      await CloudRunnerSystem.Run(`zip ${cacheKey}.zip ${path.basename(sourceFolder)}`);
+      assert(fs.existsSync(`${cacheKey}.zip`), 'cache zip exists');
+      assert(fs.existsSync(path.basename(sourceFolder)), 'source folder exists');
+      await CloudRunnerSystem.Run(`mv ${cacheKey}.zip ${cacheFolder}`);
+      RemoteClientLogger.log(`moved ${cacheKey}.zip to ${cacheFolder}`);
+      assert(fs.existsSync(`${path.join(cacheFolder, cacheKey)}.zip`), 'cache zip exists inside cache folder');
+
+      if (Input.cloudRunnerTests) {
+        await CloudRunnerSystem.Run(`ls ${cacheFolder}`);
+      }
+    } catch (error) {
+      process.chdir(`${startPath}`);
+      throw error;
+    }
+    process.chdir(`${startPath}`);
+  }
+  public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheKey: string = ``) {
+    const startPath = process.cwd();
+    RemoteClientLogger.log(`Caching for ${path.basename(destinationFolder)}`);
+    try {
+      if (!fs.existsSync(cacheFolder)) {
+        fs.mkdirSync(cacheFolder);
+      }
+
+      if (!fs.existsSync(destinationFolder)) {
+        fs.mkdirSync(destinationFolder);
+      }
+
+      const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .zip$ | head -1`))
+        .replace(/\n/g, ``)
+        .replace('.zip', '');
+
+      process.chdir(cacheFolder);
+
+      const cacheSelection = cacheKey !== `` && fs.existsSync(`${cacheKey}.zip`) ? cacheKey : latestInBranch;
+      await CloudRunnerLogger.log(`cache key ${cacheKey} selection ${cacheSelection}`);
+
+      if (fs.existsSync(`${cacheSelection}.zip`)) {
+        const resultsFolder = `results${CloudRunnerState.buildParams.buildGuid}`;
+        await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
+        if (Input.cloudRunnerTests) {
+          await CloudRunnerSystem.Run(`tree ${destinationFolder}`);
+        }
+        RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.zip`);
+        assert(`${fs.existsSync(destinationFolder)}`);
+        assert(`${fs.existsSync(`${cacheSelection}.zip`)}`);
+        const fullResultsFolder = path.join(cacheFolder, resultsFolder);
+        if (Input.cloudRunnerTests) {
+          await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
+        }
+        await CloudRunnerSystem.Run(`unzip ${cacheSelection}.zip -d ${path.basename(resultsFolder)}`);
+        RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
+        assert(`${fs.existsSync(fullResultsFolder)}`);
+        const destinationParentFolder = path.resolve(destinationFolder, '..');
+        if (fs.existsSync(destinationFolder)) {
+          fs.rmSync(destinationFolder, { recursive: true, force: true });
+        }
+        await CloudRunnerSystem.Run(
+          `mv "${fullResultsFolder}/${path.basename(destinationFolder)}" "${destinationParentFolder}"`,
+        );
+        if (Input.cloudRunnerTests) {
+          await CloudRunnerSystem.Run(`tree ${destinationParentFolder}`);
+        }
+      } else {
+        RemoteClientLogger.logWarning(`cache item ${cacheKey} doesn't exist ${destinationFolder}`);
+        if (cacheSelection !== ``) {
+          if (Input.cloudRunnerTests) {
+            await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
+          }
+          RemoteClientLogger.logWarning(`cache item ${cacheKey}.zip doesn't exist ${destinationFolder}`);
+          throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
+        }
+      }
+    } catch (error) {
+      process.chdir(`${startPath}`);
+      throw error;
+    }
+    process.chdir(`${startPath}`);
+  }
+
+  public static handleCachePurging() {
+    if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
+      RemoteClientLogger.log(`purging ${CloudRunnerState.purgeRemoteCaching}`);
+      fs.rmdirSync(CloudRunnerState.cacheFolder, { recursive: true });
+    }
+  }
+}

src/model/cli/remote-client/remote-client-services/cloud-runner-system.ts

@@ -0,0 +1,37 @@
+import { exec } from 'child_process';
+import { RemoteClientLogger } from './remote-client-logger';
+
+export class CloudRunnerSystem {
+  public static async Run(command: string, suppressError = false) {
+    for (const element of command.split(`\n`)) {
+      RemoteClientLogger.log(element);
+    }
+    return await new Promise<string>((promise) => {
+      let output = '';
+      const child = exec(command, (error, stdout, stderr) => {
+        if (error && !suppressError) {
+          throw error;
+        }
+        if (stderr) {
+          const diagnosticOutput = `${stderr.toString()}`;
+          RemoteClientLogger.logCliDiagnostic(diagnosticOutput);
+          output += diagnosticOutput;
+          return;
+        }
+        const outputChunk = `${stdout}`;
+        output += outputChunk;
+      });
+      child.on('close', function (code) {
+        RemoteClientLogger.log(`[Exit code ${code}]`);
+        if (code !== 0 && !suppressError) {
+          throw new Error(output);
+        }
+        const outputLines = output.split(`\n`);
+        for (const element of outputLines) {
+          RemoteClientLogger.log(element);
+        }
+        promise(output);
+      });
+    });
+  }
+}

src/model/cli/remote-client/remote-client-services/lfs-hashing.ts

@@ -0,0 +1,42 @@
+import path from 'path';
+import { CloudRunnerState } from '../../../cloud-runner/state/cloud-runner-state';
+import { CloudRunnerSystem } from './cloud-runner-system';
+import fs from 'fs';
+import { assert } from 'console';
+import { Input } from '../../..';
+import { RemoteClientLogger } from './remote-client-logger';
+
+export class LFSHashing {
+  public static async createLFSHashFiles() {
+    try {
+      await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
+      await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
+      assert(fs.existsSync(`.lfs-assets-guid-sum`));
+      assert(fs.existsSync(`.lfs-assets-guid`));
+      const lfsHashes = {
+        lfsGuid: fs
+          .readFileSync(`${path.join(CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8')
+          .replace(/\n/g, ``),
+        lfsGuidSum: fs
+          .readFileSync(`${path.join(CloudRunnerState.repoPathFull, `.lfs-assets-guid-sum`)}`, 'utf8')
+          .replace(/\n/g, ``),
+      };
+      if (Input.cloudRunnerTests) {
+        RemoteClientLogger.log(lfsHashes.lfsGuid);
+        RemoteClientLogger.log(lfsHashes.lfsGuidSum);
+      }
+      return lfsHashes;
+    } catch (error) {
+      throw error;
+    }
+  }
+  public static async hashAllFiles(folder: string) {
+    const startPath = process.cwd();
+    process.chdir(folder);
+    const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
+      .replace(/\n/g, '')
+      .split(` `)[0];
+    process.chdir(startPath);
+    return result;
+  }
+}

src/model/cli/remote-client/remote-client-services/remote-client-logger.ts

@@ -0,0 +1,19 @@
+import CloudRunnerLogger from '../../../cloud-runner/services/cloud-runner-logger';
+
+export class RemoteClientLogger {
+  public static log(message: string) {
+    CloudRunnerLogger.log(`[Client] ${message}`);
+  }
+
+  public static logCliError(message: string) {
+    CloudRunnerLogger.log(`[Client][Error] ${message}`);
+  }
+
+  public static logCliDiagnostic(message: string) {
+    CloudRunnerLogger.log(`[Client][Diagnostic] ${message}`);
+  }
+
+  public static logWarning(message) {
+    CloudRunnerLogger.logWarning(message);
+  }
+}

src/model/cli/remote-client/setup-cloud-runner-repository.ts

@@ -0,0 +1,81 @@
+import fs from 'fs';
+import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
+import { Caching } from './remote-client-services/caching';
+import { LFSHashing } from './remote-client-services/lfs-hashing';
+import { CloudRunnerSystem } from './remote-client-services/cloud-runner-system';
+import { Input } from '../..';
+import { RemoteClientLogger } from './remote-client-services/remote-client-logger';
+import path from 'path';
+import { assert } from 'console';
+
+export class SetupCloudRunnerRepository {
+  public static async run() {
+    try {
+      await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerState.buildPathFull}`);
+      await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerState.repoPathFull}`);
+      await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerState.cacheFolderFull}`);
+
+      process.chdir(CloudRunnerState.repoPathFull);
+      if (Input.cloudRunnerTests) {
+        await CloudRunnerSystem.Run(`ls -lh`);
+        await CloudRunnerSystem.Run(`tree`);
+      }
+      await SetupCloudRunnerRepository.cloneRepoWithoutLFSFiles();
+      if (Input.cloudRunnerTests) {
+        await CloudRunnerSystem.Run(`ls -lh`);
+        await CloudRunnerSystem.Run(`tree`);
+      }
+      const lfsHashes = await LFSHashing.createLFSHashFiles();
+      if (fs.existsSync(CloudRunnerState.libraryFolderFull)) {
+        RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
+      }
+      await Caching.PullFromCache(
+        CloudRunnerState.lfsCacheFolderFull,
+        CloudRunnerState.lfsDirectoryFull,
+        `${lfsHashes.lfsGuid}`,
+      );
+      await SetupCloudRunnerRepository.pullLatestLFS();
+      await Caching.PushToCache(
+        CloudRunnerState.lfsCacheFolderFull,
+        CloudRunnerState.lfsDirectoryFull,
+        `${lfsHashes.lfsGuid}`,
+      );
+      await Caching.PullFromCache(CloudRunnerState.libraryCacheFolderFull, CloudRunnerState.libraryFolderFull);
+      Caching.handleCachePurging();
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  private static async cloneRepoWithoutLFSFiles() {
+    try {
+      process.chdir(`${CloudRunnerState.repoPathFull}`);
+      RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
+      await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
+      RemoteClientLogger.log(`Cloning the repository being built:`);
+      await CloudRunnerSystem.Run(`git lfs install --skip-smudge`);
+      await CloudRunnerSystem.Run(
+        `git clone ${CloudRunnerState.targetBuildRepoUrl} ${path.resolve(
+          `..`,
+          path.basename(CloudRunnerState.repoPathFull),
+        )}`,
+      );
+      assert(fs.existsSync(`.git`));
+      RemoteClientLogger.log(`${CloudRunnerState.buildParams.branch}`);
+      await CloudRunnerSystem.Run(`git checkout ${CloudRunnerState.buildParams.branch}`);
+      assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
+      RemoteClientLogger.log(`Checked out ${process.env.GITHUB_SHA}`);
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  private static async pullLatestLFS() {
+    await CloudRunnerSystem.Run(`ls -lh ${CloudRunnerState.lfsDirectoryFull}/..`);
+    process.chdir(CloudRunnerState.repoPathFull);
+    await CloudRunnerSystem.Run(`git lfs pull`);
+    RemoteClientLogger.log(`pulled latest LFS files`);
+    assert(fs.existsSync(CloudRunnerState.lfsDirectoryFull));
+    await CloudRunnerSystem.Run(`ls -lh ${CloudRunnerState.lfsDirectoryFull}/..`);
+  }
+}