unity/unity-builder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cloud Runner Develop - 1.0 R.C 1 (#437)

Browse Source 
Release Candidate changeset 1 - For 1.0 Cloud Runner
This commit is contained in:
Frostebite
2022-11-07 20:41:00 +00:00
committed by GitHub
parent 96555a0945
commit 4cca069ebb
76 changed files with 5325 additions and 2023 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
src/model/cloud-runner/remote-client/index.ts
View File

@@ -9,34 +9,43 @@ import { assert } from 'console';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { CliFunction } from '../../cli/cli-functions-repository';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import YAML from 'yaml';
export class RemoteClient {
public static async bootstrapRepository() {
try {
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.repoPathAbsolute}`);
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.cacheFolderFull}`);
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
);
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
await RemoteClient.cloneRepoWithoutLFSFiles();
await RemoteClient.sizeOfFolder('repo before lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
RemoteClient.replaceLargePackageReferencesWithSharedReferences();
await RemoteClient.sizeOfFolder(
'repo before lfs cache pull',
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
);
const lfsHashes = await LfsHashing.createLFSHashFiles();
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
}
await Caching.PullFromCache(
CloudRunnerFolders.lfsCacheFolderFull,
CloudRunnerFolders.lfsFolderAbsolute,
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
await RemoteClient.pullLatestLFS();
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.PushToCache(
CloudRunnerFolders.lfsCacheFolderFull,
CloudRunnerFolders.lfsFolderAbsolute,
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await Caching.PullFromCache(CloudRunnerFolders.libraryCacheFolderFull, CloudRunnerFolders.libraryFolderAbsolute);
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
);
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.handleCachePurging();
} catch (error) {
@@ -45,37 +54,69 @@ export class RemoteClient {
}
private static async sizeOfFolder(message: string, folder: string) {
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Size of ${message}`);
await CloudRunnerSystem.Run(`du -sh ${folder}`);
}
}
private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (
CloudRunner.buildParameters.retainWorkspace &&
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
) {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
RemoteClientLogger.log(
`${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspace}`,
);
await CloudRunnerSystem.Run(`git reset --hard ${CloudRunner.buildParameters.gitSha}`);
return;
}
if (fs.existsSync(CloudRunnerFolders.repoPathAbsolute)) {
RemoteClientLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists cleaning up`);
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
}
try {
process.chdir(`${CloudRunnerFolders.repoPathAbsolute}`);
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
RemoteClientLogger.log(`Cloning the repository being built:`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
await CloudRunnerSystem.Run(
`git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.resolve(
`..`,
path.basename(CloudRunnerFolders.repoPathAbsolute),
)}`,
`git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
);
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git lfs install`);
assert(fs.existsSync(`.git`), 'git folder exists');
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
RemoteClientLogger.log(`Checked out ${process.env.GITHUB_SHA}`);
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
} catch (error) {
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
throw error;
}
}
static replaceLargePackageReferencesWithSharedReferences() {
const manifest = fs.readFileSync(
path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`),
'utf8',
);
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(manifest);
}
if (CloudRunner.buildParameters.useSharedLargePackages) {
manifest.replace(/LargePackages/g, '../../LargePackages');
}
}
private static async pullLatestLFS() {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
@@ -85,13 +126,36 @@ export class RemoteClient {
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
}
@CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`)
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async runRemoteClientJob() {
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
RemoteClientLogger.log(`Build Params:
${JSON.stringify(buildParameter, undefined, 4)}
`);
CloudRunner.buildParameters = buildParameter;
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
RemoteClient.handleRetainedWorkspace();
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await RemoteClient.bootstrapRepository();
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await RemoteClient.runCustomHookFiles(`before-build`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
}
static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = YAML.parse(fileContents.toString());
if (fileContentsObject.hook === hookLifecycle) {
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
await CloudRunnerSystem.Run(fileContentsObject.commands);
}
}
}
static handleRetainedWorkspace() {
if (!CloudRunner.buildParameters.retainWorkspace) {
return;
}
RemoteClientLogger.log(`Retained Workspace: ${CloudRunner.lockedWorkspace}`);
}
}