unity/unity-builder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cloud Runner Develop - 1.0 R.C 1 (#437)

Browse Source 
Release Candidate changeset 1 - For 1.0 Cloud Runner
This commit is contained in:
Frostebite
2022-11-07 20:41:00 +00:00
committed by GitHub
parent 96555a0945
commit 4cca069ebb
76 changed files with 5325 additions and 2023 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/model/cloud-runner/providers/aws/aws-error.ts
View File

@@ -7,7 +7,7 @@ export class AWSError {
static async handleStackCreationFailure(error: any, CF: SDK.CloudFormation, taskDefStackName: string) {
CloudRunnerLogger.log('aws error: ');
core.error(JSON.stringify(error, undefined, 4));
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log('Getting events and resources for task stack');
const events = (await CF.describeStackEvents({ StackName: taskDefStackName }).promise()).StackEvents;
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));

6
src/model/cloud-runner/providers/aws/aws-job-stack.ts
View File

@@ -69,6 +69,7 @@ export class AWSJobStack {
const secretsMappedToCloudFormationParameters = secrets.map((x) => {
return { ParameterKey: x.ParameterKey.replace(/[^\dA-Za-z]/g, ''), ParameterValue: x.ParameterValue };
});
const logGroupName = `${this.baseStackName}/${taskDefStackName}`;
const parameters = [
{
ParameterKey: 'EnvironmentName',
@@ -82,6 +83,10 @@ export class AWSJobStack {
ParameterKey: 'ServiceName',
ParameterValue: taskDefStackName,
},
{
ParameterKey: 'LogGroupName',
ParameterValue: logGroupName,
},
{
ParameterKey: 'Command',
ParameterValue: 'echo "this template should be overwritten when running a task"',
@@ -115,6 +120,7 @@ export class AWSJobStack {
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
previousStackExists = true;
CloudRunnerLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
await new Promise((promise) => setTimeout(promise, 5000));
}
}
}

109
src/model/cloud-runner/providers/aws/aws-task-runner.ts
View File

@@ -6,18 +6,18 @@ import * as zlib from 'zlib';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import { Input } from '../../..';
import CloudRunner from '../../cloud-runner';
import { CloudRunnerBuildCommandProcessor } from '../../services/cloud-runner-build-command-process';
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
import { FollowLogStreamService } from '../../services/follow-log-stream-service';
import CloudRunnerOptions from '../../cloud-runner-options';
class AWSTaskRunner {
public static ECS: AWS.ECS;
public static Kinesis: AWS.Kinesis;
static async runTask(
taskDef: CloudRunnerAWSTaskDef,
ECS: AWS.ECS,
CF: AWS.CloudFormation,
environment: CloudRunnerEnvironmentVariable[],
buildGuid: string,
commands: string,
) {
): Promise<{ output: string; shouldCleanup: boolean }> {
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
const taskDefinition =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId || '';
@@ -30,7 +30,7 @@ class AWSTaskRunner {
const streamName =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
const task = await ECS.runTask({
const task = await AWSTaskRunner.ECS.runTask({
cluster,
taskDefinition,
platformVersion: '1.4.0',
@@ -39,7 +39,7 @@ class AWSTaskRunner {
{
name: taskDef.taskDefStackName,
environment,
command: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(commands, CloudRunner.buildParameters)],
command: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
},
],
},
@@ -54,20 +54,25 @@ class AWSTaskRunner {
}).promise();
const taskArn = task.tasks?.[0].taskArn || '';
CloudRunnerLogger.log('Cloud runner job is starting');
await AWSTaskRunner.waitUntilTaskRunning(ECS, taskArn, cluster);
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
CloudRunnerLogger.log(
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(ECS, cluster, taskArn))?.lastStatus}`,
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus}`,
);
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(
ECS,
CF,
taskDef,
cluster,
taskArn,
streamName,
);
const taskData = await AWSTaskRunner.describeTasks(ECS, cluster, taskArn);
const exitCode = taskData.containers?.[0].exitCode;
if (!CloudRunnerOptions.watchCloudRunnerToEnd) {
const shouldCleanup: boolean = false;
const output: string = '';
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
return { output, shouldCleanup };
}
CloudRunnerLogger.log(`Streaming...`);
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
await new Promise((resolve) => resolve(5000));
const taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
const containerState = taskData.containers?.[0];
const exitCode = containerState?.exitCode || undefined;
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING');
if (wasSuccessful) {
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
@@ -85,15 +90,15 @@ class AWSTaskRunner {
}
}
private static async waitUntilTaskRunning(ECS: AWS.ECS, taskArn: string, cluster: string) {
private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
try {
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
await AWSTaskRunner.ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
} catch (error_) {
const error = error_ as Error;
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(
`Cloud runner job has ended ${
(await AWSTaskRunner.describeTasks(ECS, cluster, taskArn)).containers?.[0].lastStatus
(await AWSTaskRunner.describeTasks(cluster, taskArn)).containers?.[0].lastStatus
}`,
);
@@ -102,8 +107,8 @@ class AWSTaskRunner {
}
}
static async describeTasks(ECS: AWS.ECS, clusterName: string, taskArn: string) {
const tasks = await ECS.describeTasks({
static async describeTasks(clusterName: string, taskArn: string) {
const tasks = await AWSTaskRunner.ECS.describeTasks({
cluster: clusterName,
tasks: [taskArn],
}).promise();
@@ -114,17 +119,11 @@ class AWSTaskRunner {
}
}
static async streamLogsUntilTaskStops(
ECS: AWS.ECS,
CF: AWS.CloudFormation,
taskDef: CloudRunnerAWSTaskDef,
clusterName: string,
taskArn: string,
kinesisStreamName: string,
) {
const kinesis = new AWS.Kinesis();
const stream = await AWSTaskRunner.getLogStream(kinesis, kinesisStreamName);
let iterator = await AWSTaskRunner.getLogIterator(kinesis, stream);
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(`Streaming...`);
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
let iterator = await AWSTaskRunner.getLogIterator(stream);
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
@@ -134,13 +133,11 @@ class AWSTaskRunner {
let output = '';
while (shouldReadLogs) {
await new Promise((resolve) => setTimeout(resolve, 1500));
const taskData = await AWSTaskRunner.describeTasks(ECS, clusterName, taskArn);
const taskData = await AWSTaskRunner.describeTasks(clusterName, taskArn);
({ timestamp, shouldReadLogs } = AWSTaskRunner.checkStreamingShouldContinue(taskData, timestamp, shouldReadLogs));
({ iterator, shouldReadLogs, output, shouldCleanup } = await AWSTaskRunner.handleLogStreamIteration(
kinesis,
iterator,
shouldReadLogs,
taskDef,
output,
shouldCleanup,
));
@@ -150,23 +147,18 @@ class AWSTaskRunner {
}
private static async handleLogStreamIteration(
kinesis: AWS.Kinesis,
iterator: string,
shouldReadLogs: boolean,
taskDef: CloudRunnerAWSTaskDef,
output: string,
shouldCleanup: boolean,
) {
const records = await kinesis
.getRecords({
ShardIterator: iterator,
})
.promise();
const records = await AWSTaskRunner.Kinesis.getRecords({
ShardIterator: iterator,
}).promise();
iterator = records.NextShardIterator || '';
({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords(
records,
iterator,
taskDef,
shouldReadLogs,
output,
shouldCleanup,
@@ -197,7 +189,6 @@ class AWSTaskRunner {
private static logRecords(
records,
iterator: string,
taskDef: CloudRunnerAWSTaskDef,
shouldReadLogs: boolean,
output: string,
shouldCleanup: boolean,
@@ -224,24 +215,20 @@ class AWSTaskRunner {
return { shouldReadLogs, output, shouldCleanup };
}
private static async getLogStream(kinesis: AWS.Kinesis, kinesisStreamName: string) {
return await kinesis
.describeStream({
StreamName: kinesisStreamName,
})
.promise();
private static async getLogStream(kinesisStreamName: string) {
return await AWSTaskRunner.Kinesis.describeStream({
StreamName: kinesisStreamName,
}).promise();
}
private static async getLogIterator(kinesis: AWS.Kinesis, stream) {
private static async getLogIterator(stream) {
return (
(
await kinesis
.getShardIterator({
ShardIteratorType: 'TRIM_HORIZON',
StreamName: stream.StreamDescription.StreamName,
ShardId: stream.StreamDescription.Shards[0].ShardId,
})
.promise()
await AWSTaskRunner.Kinesis.getShardIterator({
ShardIteratorType: 'TRIM_HORIZON',
StreamName: stream.StreamDescription.StreamName,
ShardId: stream.StreamDescription.Shards[0].ShardId,
}).promise()
).ShardIterator || ''
);
}

10
src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts
View File

@@ -11,6 +11,10 @@ Parameters:
Type: String
Default: example
Description: A name for the service
LogGroupName:
Type: String
Default: example
Description: Name to use for the log group created for this task
ImageUrl:
Type: String
Default: nginx
@@ -68,7 +72,7 @@ Resources:
LogGroup:
Type: 'AWS::Logs::LogGroup'
Properties:
LogGroupName: !Ref ServiceName
LogGroupName: !Ref LogGroupName
Metadata:
'AWS::CloudFormation::Designer':
id: aece53ae-b82d-4267-bc16-ed964b05db27
@@ -78,7 +82,7 @@ Resources:
FilterPattern: ''
RoleArn:
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:CloudWatchIAMRole'
LogGroupName: !Ref ServiceName
LogGroupName: !Ref LogGroupName
DestinationArn:
'Fn::GetAtt':
- KinesisStream
@@ -147,7 +151,7 @@ Resources:
LogConfiguration:
LogDriver: awslogs
Options:
awslogs-group: !Ref ServiceName
awslogs-group: !Ref LogGroupName
awslogs-region: !Ref 'AWS::Region'
awslogs-stream-prefix: !Ref ServiceName
DependsOn:

170
src/model/cloud-runner/providers/aws/commands/aws-cli-commands.ts
View File

@@ -1,170 +0,0 @@
import AWS from 'aws-sdk';
import { CliFunction } from '../../../../cli/cli-functions-repository';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
export class AwsCliCommands {
@CliFunction(`aws-list-all`, `List all resources`)
static async awsListAll() {
await AwsCliCommands.awsListStacks(undefined, true);
await AwsCliCommands.awsListTasks();
await AwsCliCommands.awsListLogGroups(undefined, true);
}
@CliFunction(`aws-garbage-collect`, `garbage collect aws resources not in use !WIP!`)
static async garbageCollectAws() {
await AwsCliCommands.cleanup(false);
}
@CliFunction(`aws-garbage-collect-all`, `garbage collect aws resources regardless of whether they are in use`)
static async garbageCollectAwsAll() {
await AwsCliCommands.cleanup(true);
}
@CliFunction(
`aws-garbage-collect-all-1d-older`,
`garbage collect aws resources created more than 1d ago (ignore if they are in use)`,
)
static async garbageCollectAwsAllOlderThanOneDay() {
await AwsCliCommands.cleanup(true, true);
}
static isOlderThan1day(date: any) {
const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0;
}
@CliFunction(`aws-list-stacks`, `List stacks`)
static async awsListStacks(perResultCallback: any = false, verbose: boolean = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
// _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate = new Date(element.CreationTime.getTime() - Date.now());
if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${ageDate.getDay()} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
const baseStacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
const ageDate = new Date(element.CreationTime.getTime() - Date.now());
if (verbose)
CloudRunnerLogger.log(
`Base Stack ${
element.StackName
} - Age D${ageDate.getHours()} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
if (stacks === undefined) {
return;
}
}
@CliFunction(`aws-list-tasks`, `List tasks`)
static async awsListTasks(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.ECS();
const clusters = (await ecs.listClusters().promise()).clusterArns || [];
CloudRunnerLogger.log(`Clusters ${clusters.length}`);
for (const element of clusters) {
const input: AWS.ECS.ListTasksRequest = {
cluster: element,
};
const list = (await ecs.listTasks(input).promise()).taskArns || [];
if (list.length > 0) {
const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList === []) {
continue;
}
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) {
if (taskElement === undefined) {
continue;
}
taskElement.overrides = {};
taskElement.attachments = [];
if (taskElement.createdAt === undefined) {
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
if (perResultCallback) await perResultCallback(taskElement, element);
}
}
}
}
@CliFunction(`aws-list-log-groups`, `List tasks`)
static async awsListLogGroups(perResultCallback: any = false, verbose: boolean = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.CloudWatchLogs();
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate = new Date(new Date(element.creationTime).getTime() - Date.now());
if (verbose)
CloudRunnerLogger.log(
`Log Group Name ${
element.logGroupName
} - Age D${ageDate.getDay()} H${ageDate.getHours()} M${ageDate.getMinutes()} - 1d old ${AwsCliCommands.isOlderThan1day(
new Date(element.creationTime),
)}`,
);
if (perResultCallback) await perResultCallback(element, element);
}
}
private static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const ecs = new AWS.ECS();
const cwl = new AWS.CloudWatchLogs();
await AwsCliCommands.awsListStacks(async (element) => {
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(element.CreationTime))) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
return;
}
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
await CF.deleteStack(deleteStackInput).promise();
}
});
await AwsCliCommands.awsListTasks(async (taskElement, element) => {
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(taskElement.CreatedAt))) {
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
}
});
await AwsCliCommands.awsListLogGroups(async (element) => {
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(new Date(element.createdAt)))) {
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
}
});
}
}

62
src/model/cloud-runner/providers/aws/index.ts
View File

@@ -2,13 +2,18 @@ import * as SDK from 'aws-sdk';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import AWSTaskRunner from './aws-task-runner';
import AwsTaskRunner from './aws-task-runner';
import { ProviderInterface } from '../provider-interface';
import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import { AWSJobStack } from './aws-job-stack';
import { AWSBaseStack } from './aws-base-stack';
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
import { Input } from '../../..';
import { TertiaryResourcesService } from './services/tertiary-resources-service';
import { GarbageCollectionService } from './services/garbage-collection-service';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { TaskService } from './services/task-service';
class AWSBuildEnvironment implements ProviderInterface {
private baseStackName: string;
@@ -16,7 +21,43 @@ class AWSBuildEnvironment implements ProviderInterface {
constructor(buildParameters: BuildParameters) {
this.baseStackName = buildParameters.awsBaseStackName;
}
async cleanup(
async listResources(): Promise<ProviderResource[]> {
await TaskService.awsListJobs();
await TertiaryResourcesService.awsListLogGroups();
await TaskService.awsListTasks();
await TaskService.awsListStacks();
return [];
}
listWorkflow(): Promise<ProviderWorkflow[]> {
throw new Error('Method not implemented.');
}
async watchWorkflow(): Promise<string> {
return await TaskService.watch();
}
async listOtherResources(): Promise<string> {
await TertiaryResourcesService.awsListLogGroups();
return '';
}
async garbageCollect(
filter: string,
previewOnly: boolean,
// eslint-disable-next-line no-unused-vars
olderThan: Number,
// eslint-disable-next-line no-unused-vars
fullCache: boolean,
// eslint-disable-next-line no-unused-vars
baseDependencies: boolean,
): Promise<string> {
await GarbageCollectionService.cleanup(!previewOnly);
return ``;
}
async cleanupWorkflow(
// eslint-disable-next-line no-unused-vars
buildGuid: string,
// eslint-disable-next-line no-unused-vars
@@ -26,7 +67,7 @@ class AWSBuildEnvironment implements ProviderInterface {
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
async setup(
async setupWorkflow(
// eslint-disable-next-line no-unused-vars
buildGuid: string,
// eslint-disable-next-line no-unused-vars
@@ -37,7 +78,7 @@ class AWSBuildEnvironment implements ProviderInterface {
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
async runTask(
async runTaskInWorkflow(
buildGuid: string,
image: string,
commands: string,
@@ -49,12 +90,14 @@ class AWSBuildEnvironment implements ProviderInterface {
process.env.AWS_REGION = Input.region;
const ECS = new SDK.ECS();
const CF = new SDK.CloudFormation();
AwsTaskRunner.ECS = ECS;
AwsTaskRunner.Kinesis = new SDK.Kinesis();
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now();
await new AWSBaseStack(this.baseStackName).setupBaseStack(CF);
const taskDef = await new AWSJobStack(this.baseStackName).setupCloudFormations(
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
CF,
buildGuid,
image,
@@ -69,7 +112,7 @@ class AWSBuildEnvironment implements ProviderInterface {
try {
const postSetupStacksTimeMs = Date.now();
CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
const { output, shouldCleanup } = await AWSTaskRunner.runTask(taskDef, ECS, CF, environment, buildGuid, commands);
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
postRunTaskTimeMs = Date.now();
CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
if (shouldCleanup) {
@@ -81,6 +124,7 @@ class AWSBuildEnvironment implements ProviderInterface {
return output;
} catch (error) {
CloudRunnerLogger.log(`error running task ${error}`);
await this.cleanupResources(CF, taskDef);
throw error;
}

58
src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts Normal file
View File

@@ -0,0 +1,58 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { TaskService } from './task-service';
import { TertiaryResourcesService } from './tertiary-resources-service';
export class GarbageCollectionService {
static isOlderThan1day(date: any) {
const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0;
}
public static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const ecs = new AWS.ECS();
const cwl = new AWS.CloudWatchLogs();
const taskDefinitionsInUse = new Array();
await TaskService.awsListTasks(async (taskElement, element) => {
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
}
});
await TaskService.awsListStacks(async (element) => {
if (
(await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some(
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
)
) {
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
return;
}
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(element.CreationTime))) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
return;
}
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
await CF.deleteStack(deleteStackInput).promise();
}
});
await TertiaryResourcesService.awsListLogGroups(async (element) => {
if (
deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))
) {
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
}
});
}
}

131
src/model/cloud-runner/providers/aws/services/task-service.ts Normal file
View File

@@ -0,0 +1,131 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner';
export class TaskService {
static async watch() {
// eslint-disable-next-line no-unused-vars
const { output, shouldCleanup } = await AwsTaskRunner.streamLogsUntilTaskStops(
process.env.cluster || ``,
process.env.taskArn || ``,
process.env.streamName || ``,
);
return output;
}
public static async awsListStacks(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
// _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
const baseStacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
if (stacks === undefined) {
return;
}
}
public static async awsListTasks(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.ECS();
const clusters = (await ecs.listClusters().promise()).clusterArns || [];
CloudRunnerLogger.log(`Clusters ${clusters.length}`);
for (const element of clusters) {
const input: AWS.ECS.ListTasksRequest = {
cluster: element,
};
const list = (await ecs.listTasks(input).promise()).taskArns || [];
if (list.length > 0) {
const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList.length === 0) {
continue;
}
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) {
if (taskElement === undefined) {
continue;
}
taskElement.overrides = {};
taskElement.attachments = [];
if (taskElement.createdAt === undefined) {
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
if (perResultCallback) await perResultCallback(taskElement, element);
}
}
}
}
public static async awsListJobs(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
}
public static async awsDescribeJob(job: string) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stack = (await CF.listStacks().promise()).StackSummaries?.find((_x) => _x.StackName === job) || undefined;
const stackInfo = (await CF.describeStackResources({ StackName: job }).promise()) || undefined;
const stackInfo2 = (await CF.describeStacks({ StackName: job }).promise()) || undefined;
if (stack === undefined) {
throw new Error('stack not defined');
}
const ageDate: Date = new Date(Date.now() - stack.CreationTime.getTime());
const message = `
Task Stack ${stack.StackName}
Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}
${JSON.stringify(stack, undefined, 4)}
${JSON.stringify(stackInfo, undefined, 4)}
${JSON.stringify(stackInfo2, undefined, 4)}
`;
CloudRunnerLogger.log(message);
return message;
}
}

36
src/model/cloud-runner/providers/aws/services/tertiary-resources-service.ts Normal file
View File

@@ -0,0 +1,36 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
export class TertiaryResourcesService {
public static async awsListLogGroups(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.CloudWatchLogs();
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element, element);
}
}
}