Cloud Runner Develop - 1.0 R.C 1 (#437)
Release Candidate changeset 1 - For 1.0 Cloud Runner
This commit is contained in:
Frostebite
GitHub
@@ -1,63 +0,0 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import Input from '../../input';
|
||||
import UnityVersioning from '../../unity-versioning';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import { CloudRunnerSystem } from '../services/cloud-runner-system';
|
||||
import { Caching } from './caching';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
describe('Cloud Runner Caching', () => {
|
||||
it('responds', () => {});
|
||||
});
|
||||
describe('Cloud Runner Caching', () => {
|
||||
if (process.platform === 'linux') {
|
||||
it('Simple caching works', async () => {
|
||||
Cli.options = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.read('test-project'),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
};
|
||||
Input.githubInputEnabled = false;
|
||||
const buildParameter = await BuildParameters.create();
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
|
||||
// Create test folder
|
||||
const testFolder = path.resolve(__dirname, Cli.options.cacheKey);
|
||||
fs.mkdirSync(testFolder);
|
||||
|
||||
// Create cache folder
|
||||
const cacheFolder = path.resolve(__dirname, `cache-${Cli.options.cacheKey}`);
|
||||
fs.mkdirSync(cacheFolder);
|
||||
|
||||
// Add test file to test folders
|
||||
fs.writeFileSync(path.resolve(testFolder, 'test.txt'), Cli.options.cacheKey);
|
||||
await Caching.PushToCache(cacheFolder, testFolder, `${Cli.options.cacheKey}`);
|
||||
|
||||
// Delete test folder
|
||||
fs.rmdirSync(testFolder, { recursive: true });
|
||||
await Caching.PullFromCache(
|
||||
cacheFolder.replace(/\\/g, `/`),
|
||||
testFolder.replace(/\\/g, `/`),
|
||||
`${Cli.options.cacheKey}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`du -h ${__dirname}`);
|
||||
await CloudRunnerSystem.Run(`tree ${testFolder}`);
|
||||
await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
|
||||
|
||||
// Compare validity to original hash
|
||||
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(
|
||||
Cli.options.cacheKey,
|
||||
);
|
||||
fs.rmdirSync(testFolder, { recursive: true });
|
||||
fs.rmdirSync(cacheFolder, { recursive: true });
|
||||
|
||||
Input.githubInputEnabled = true;
|
||||
delete Cli.options;
|
||||
}, 1000000);
|
||||
}
|
||||
});
|
||||
@@ -46,40 +46,49 @@ export class Caching {
|
||||
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const startPath = process.cwd();
|
||||
const compressionSuffix = CloudRunner.buildParameters.useLz4Compression ? '.lz4' : '';
|
||||
try {
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||
}
|
||||
process.chdir(path.resolve(sourceFolder, '..'));
|
||||
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log(
|
||||
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
|
||||
sourceFolder,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
// eslint-disable-next-line func-style
|
||||
const formatFunction = function (format: string) {
|
||||
const arguments_ = Array.prototype.slice.call(
|
||||
[path.resolve(sourceFolder, '..'), cacheFolder, cacheArtifactName],
|
||||
1,
|
||||
);
|
||||
const contents = await fs.promises.readdir(path.basename(sourceFolder));
|
||||
CloudRunnerLogger.log(
|
||||
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
|
||||
);
|
||||
|
||||
return format.replace(/{(\d+)}/g, function (match, number) {
|
||||
return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
|
||||
});
|
||||
};
|
||||
await CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar ${path.basename(sourceFolder)}`);
|
||||
assert(await fileExists(`${cacheArtifactName}.tar`), 'cache archive exists');
|
||||
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
|
||||
if (CloudRunner.buildParameters.cachePushOverrideCommand) {
|
||||
await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePushOverrideCommand));
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
// await CloudRunnerSystem.Run(`tree -L 2 ./..`);
|
||||
// await CloudRunnerSystem.Run(`tree -L 2`);
|
||||
}
|
||||
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar ${cacheFolder}`);
|
||||
|
||||
if (contents.length === 0) {
|
||||
CloudRunnerLogger.log(
|
||||
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
|
||||
);
|
||||
process.chdir(`${startPath}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await CloudRunnerSystem.Run(
|
||||
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} ${path.basename(sourceFolder)}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
|
||||
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
|
||||
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
|
||||
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
|
||||
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
|
||||
assert(
|
||||
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar`),
|
||||
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
|
||||
'cache archive exists inside cache folder',
|
||||
);
|
||||
} catch (error) {
|
||||
@@ -90,6 +99,7 @@ export class Caching {
|
||||
}
|
||||
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const compressionSuffix = CloudRunner.buildParameters.useLz4Compression ? '.lz4' : '';
|
||||
const startPath = process.cwd();
|
||||
RemoteClientLogger.log(`Caching for ${path.basename(destinationFolder)}`);
|
||||
try {
|
||||
@@ -101,38 +111,26 @@ export class Caching {
|
||||
await fs.promises.mkdir(destinationFolder);
|
||||
}
|
||||
|
||||
const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar$ | head -1`))
|
||||
const latestInBranch = await (
|
||||
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
|
||||
)
|
||||
.replace(/\n/g, ``)
|
||||
.replace('.tar', '');
|
||||
.replace(`.tar${compressionSuffix}`, '');
|
||||
|
||||
process.chdir(cacheFolder);
|
||||
|
||||
const cacheSelection =
|
||||
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar`)) ? cacheArtifactName : latestInBranch;
|
||||
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
|
||||
? cacheArtifactName
|
||||
: latestInBranch;
|
||||
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||
|
||||
// eslint-disable-next-line func-style
|
||||
const formatFunction = function (format: string) {
|
||||
const arguments_ = Array.prototype.slice.call(
|
||||
[path.resolve(destinationFolder, '..'), cacheFolder, cacheArtifactName],
|
||||
1,
|
||||
);
|
||||
|
||||
return format.replace(/{(\d+)}/g, function (match, number) {
|
||||
return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
|
||||
});
|
||||
};
|
||||
|
||||
if (CloudRunner.buildParameters.cachePullOverrideCommand) {
|
||||
await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePullOverrideCommand));
|
||||
}
|
||||
|
||||
if (await fileExists(`${cacheSelection}.tar`)) {
|
||||
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
||||
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar`);
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
||||
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
||||
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar -C ${fullResultsFolder}`);
|
||||
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
||||
@@ -152,15 +150,17 @@ export class Caching {
|
||||
} else {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
|
||||
if (cacheSelection !== ``) {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar doesn't exist ${destinationFolder}`);
|
||||
RemoteClientLogger.logWarning(
|
||||
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
|
||||
);
|
||||
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
process.chdir(`${startPath}`);
|
||||
process.chdir(startPath);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(`${startPath}`);
|
||||
process.chdir(startPath);
|
||||
}
|
||||
|
||||
public static async handleCachePurging() {
|
||||
|
||||
@@ -9,34 +9,43 @@ import { assert } from 'console';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
import { CloudRunnerSystem } from '../services/cloud-runner-system';
|
||||
import YAML from 'yaml';
|
||||
|
||||
export class RemoteClient {
|
||||
public static async bootstrapRepository() {
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.repoPathAbsolute}`);
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.cacheFolderFull}`);
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
|
||||
);
|
||||
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
|
||||
await RemoteClient.cloneRepoWithoutLFSFiles();
|
||||
await RemoteClient.sizeOfFolder('repo before lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
||||
await RemoteClient.sizeOfFolder(
|
||||
'repo before lfs cache pull',
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
|
||||
);
|
||||
const lfsHashes = await LfsHashing.createLFSHashFiles();
|
||||
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
|
||||
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
|
||||
}
|
||||
await Caching.PullFromCache(
|
||||
CloudRunnerFolders.lfsCacheFolderFull,
|
||||
CloudRunnerFolders.lfsFolderAbsolute,
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
|
||||
`${lfsHashes.lfsGuidSum}`,
|
||||
);
|
||||
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await RemoteClient.pullLatestLFS();
|
||||
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await Caching.PushToCache(
|
||||
CloudRunnerFolders.lfsCacheFolderFull,
|
||||
CloudRunnerFolders.lfsFolderAbsolute,
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
|
||||
`${lfsHashes.lfsGuidSum}`,
|
||||
);
|
||||
await Caching.PullFromCache(CloudRunnerFolders.libraryCacheFolderFull, CloudRunnerFolders.libraryFolderAbsolute);
|
||||
await Caching.PullFromCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
||||
);
|
||||
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await Caching.handleCachePurging();
|
||||
} catch (error) {
|
||||
@@ -45,37 +54,69 @@ export class RemoteClient {
|
||||
}
|
||||
|
||||
private static async sizeOfFolder(message: string, folder: string) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log(`Size of ${message}`);
|
||||
await CloudRunnerSystem.Run(`du -sh ${folder}`);
|
||||
}
|
||||
}
|
||||
|
||||
private static async cloneRepoWithoutLFSFiles() {
|
||||
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
|
||||
if (
|
||||
CloudRunner.buildParameters.retainWorkspace &&
|
||||
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
|
||||
) {
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
RemoteClientLogger.log(
|
||||
`${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspace}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`git reset --hard ${CloudRunner.buildParameters.gitSha}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (fs.existsSync(CloudRunnerFolders.repoPathAbsolute)) {
|
||||
RemoteClientLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists cleaning up`);
|
||||
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
|
||||
}
|
||||
|
||||
try {
|
||||
process.chdir(`${CloudRunnerFolders.repoPathAbsolute}`);
|
||||
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
||||
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
|
||||
RemoteClientLogger.log(`Cloning the repository being built:`);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.resolve(
|
||||
`..`,
|
||||
path.basename(CloudRunnerFolders.repoPathAbsolute),
|
||||
)}`,
|
||||
`git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
|
||||
);
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
await CloudRunnerSystem.Run(`git lfs install`);
|
||||
assert(fs.existsSync(`.git`), 'git folder exists');
|
||||
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
|
||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
|
||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
|
||||
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
||||
RemoteClientLogger.log(`Checked out ${process.env.GITHUB_SHA}`);
|
||||
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
|
||||
} catch (error) {
|
||||
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
static replaceLargePackageReferencesWithSharedReferences() {
|
||||
const manifest = fs.readFileSync(
|
||||
path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`),
|
||||
'utf8',
|
||||
);
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log(manifest);
|
||||
}
|
||||
if (CloudRunner.buildParameters.useSharedLargePackages) {
|
||||
manifest.replace(/LargePackages/g, '../../LargePackages');
|
||||
}
|
||||
}
|
||||
|
||||
private static async pullLatestLFS() {
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
|
||||
@@ -85,13 +126,36 @@ export class RemoteClient {
|
||||
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
|
||||
}
|
||||
|
||||
@CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`)
|
||||
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
||||
static async runRemoteClientJob() {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
RemoteClientLogger.log(`Build Params:
|
||||
${JSON.stringify(buildParameter, undefined, 4)}
|
||||
`);
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
RemoteClient.handleRetainedWorkspace();
|
||||
|
||||
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
await RemoteClient.bootstrapRepository();
|
||||
|
||||
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
await RemoteClient.runCustomHookFiles(`before-build`);
|
||||
|
||||
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
}
|
||||
static async runCustomHookFiles(hookLifecycle: string) {
|
||||
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
|
||||
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
|
||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
||||
for (const file of files) {
|
||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
||||
const fileContentsObject = YAML.parse(fileContents.toString());
|
||||
if (fileContentsObject.hook === hookLifecycle) {
|
||||
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
|
||||
await CloudRunnerSystem.Run(fileContentsObject.commands);
|
||||
}
|
||||
}
|
||||
}
|
||||
static handleRetainedWorkspace() {
|
||||
if (!CloudRunner.buildParameters.retainWorkspace) {
|
||||
return;
|
||||
}
|
||||
RemoteClientLogger.log(`Retained Workspace: ${CloudRunner.lockedWorkspace}`);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user