Cloud Runner Develop - 1.0 R.C 1 (#437)
Release Candidate changeset 1 - For 1.0 Cloud Runner
This commit is contained in:
Frostebite
GitHub
@@ -7,7 +7,7 @@ export class AWSError {
|
||||
static async handleStackCreationFailure(error: any, CF: SDK.CloudFormation, taskDefStackName: string) {
|
||||
CloudRunnerLogger.log('aws error: ');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log('Getting events and resources for task stack');
|
||||
const events = (await CF.describeStackEvents({ StackName: taskDefStackName }).promise()).StackEvents;
|
||||
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));
|
||||
|
||||
@@ -69,6 +69,7 @@ export class AWSJobStack {
|
||||
const secretsMappedToCloudFormationParameters = secrets.map((x) => {
|
||||
return { ParameterKey: x.ParameterKey.replace(/[^\dA-Za-z]/g, ''), ParameterValue: x.ParameterValue };
|
||||
});
|
||||
const logGroupName = `${this.baseStackName}/${taskDefStackName}`;
|
||||
const parameters = [
|
||||
{
|
||||
ParameterKey: 'EnvironmentName',
|
||||
@@ -82,6 +83,10 @@ export class AWSJobStack {
|
||||
ParameterKey: 'ServiceName',
|
||||
ParameterValue: taskDefStackName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'LogGroupName',
|
||||
ParameterValue: logGroupName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'Command',
|
||||
ParameterValue: 'echo "this template should be overwritten when running a task"',
|
||||
@@ -115,6 +120,7 @@ export class AWSJobStack {
|
||||
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
|
||||
previousStackExists = true;
|
||||
CloudRunnerLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,18 +6,18 @@ import * as zlib from 'zlib';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { Input } from '../../..';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../../services/cloud-runner-build-command-process';
|
||||
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
|
||||
import { FollowLogStreamService } from '../../services/follow-log-stream-service';
|
||||
import CloudRunnerOptions from '../../cloud-runner-options';
|
||||
|
||||
class AWSTaskRunner {
|
||||
public static ECS: AWS.ECS;
|
||||
public static Kinesis: AWS.Kinesis;
|
||||
static async runTask(
|
||||
taskDef: CloudRunnerAWSTaskDef,
|
||||
ECS: AWS.ECS,
|
||||
CF: AWS.CloudFormation,
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
buildGuid: string,
|
||||
commands: string,
|
||||
) {
|
||||
): Promise<{ output: string; shouldCleanup: boolean }> {
|
||||
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||
const taskDefinition =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId || '';
|
||||
@@ -30,7 +30,7 @@ class AWSTaskRunner {
|
||||
const streamName =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
|
||||
|
||||
const task = await ECS.runTask({
|
||||
const task = await AWSTaskRunner.ECS.runTask({
|
||||
cluster,
|
||||
taskDefinition,
|
||||
platformVersion: '1.4.0',
|
||||
@@ -39,7 +39,7 @@ class AWSTaskRunner {
|
||||
{
|
||||
name: taskDef.taskDefStackName,
|
||||
environment,
|
||||
command: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(commands, CloudRunner.buildParameters)],
|
||||
command: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -54,20 +54,25 @@ class AWSTaskRunner {
|
||||
}).promise();
|
||||
const taskArn = task.tasks?.[0].taskArn || '';
|
||||
CloudRunnerLogger.log('Cloud runner job is starting');
|
||||
await AWSTaskRunner.waitUntilTaskRunning(ECS, taskArn, cluster);
|
||||
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
|
||||
CloudRunnerLogger.log(
|
||||
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(ECS, cluster, taskArn))?.lastStatus}`,
|
||||
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus}`,
|
||||
);
|
||||
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(
|
||||
ECS,
|
||||
CF,
|
||||
taskDef,
|
||||
cluster,
|
||||
taskArn,
|
||||
streamName,
|
||||
);
|
||||
const taskData = await AWSTaskRunner.describeTasks(ECS, cluster, taskArn);
|
||||
const exitCode = taskData.containers?.[0].exitCode;
|
||||
if (!CloudRunnerOptions.watchCloudRunnerToEnd) {
|
||||
const shouldCleanup: boolean = false;
|
||||
const output: string = '';
|
||||
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
|
||||
|
||||
return { output, shouldCleanup };
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`Streaming...`);
|
||||
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
|
||||
await new Promise((resolve) => resolve(5000));
|
||||
const taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
|
||||
const containerState = taskData.containers?.[0];
|
||||
const exitCode = containerState?.exitCode || undefined;
|
||||
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
|
||||
const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING');
|
||||
if (wasSuccessful) {
|
||||
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
|
||||
@@ -85,15 +90,15 @@ class AWSTaskRunner {
|
||||
}
|
||||
}
|
||||
|
||||
private static async waitUntilTaskRunning(ECS: AWS.ECS, taskArn: string, cluster: string) {
|
||||
private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
|
||||
try {
|
||||
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
|
||||
await AWSTaskRunner.ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
|
||||
} catch (error_) {
|
||||
const error = error_ as Error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
CloudRunnerLogger.log(
|
||||
`Cloud runner job has ended ${
|
||||
(await AWSTaskRunner.describeTasks(ECS, cluster, taskArn)).containers?.[0].lastStatus
|
||||
(await AWSTaskRunner.describeTasks(cluster, taskArn)).containers?.[0].lastStatus
|
||||
}`,
|
||||
);
|
||||
|
||||
@@ -102,8 +107,8 @@ class AWSTaskRunner {
|
||||
}
|
||||
}
|
||||
|
||||
static async describeTasks(ECS: AWS.ECS, clusterName: string, taskArn: string) {
|
||||
const tasks = await ECS.describeTasks({
|
||||
static async describeTasks(clusterName: string, taskArn: string) {
|
||||
const tasks = await AWSTaskRunner.ECS.describeTasks({
|
||||
cluster: clusterName,
|
||||
tasks: [taskArn],
|
||||
}).promise();
|
||||
@@ -114,17 +119,11 @@ class AWSTaskRunner {
|
||||
}
|
||||
}
|
||||
|
||||
static async streamLogsUntilTaskStops(
|
||||
ECS: AWS.ECS,
|
||||
CF: AWS.CloudFormation,
|
||||
taskDef: CloudRunnerAWSTaskDef,
|
||||
clusterName: string,
|
||||
taskArn: string,
|
||||
kinesisStreamName: string,
|
||||
) {
|
||||
const kinesis = new AWS.Kinesis();
|
||||
const stream = await AWSTaskRunner.getLogStream(kinesis, kinesisStreamName);
|
||||
let iterator = await AWSTaskRunner.getLogIterator(kinesis, stream);
|
||||
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
CloudRunnerLogger.log(`Streaming...`);
|
||||
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
|
||||
let iterator = await AWSTaskRunner.getLogIterator(stream);
|
||||
|
||||
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
|
||||
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
|
||||
@@ -134,13 +133,11 @@ class AWSTaskRunner {
|
||||
let output = '';
|
||||
while (shouldReadLogs) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
const taskData = await AWSTaskRunner.describeTasks(ECS, clusterName, taskArn);
|
||||
const taskData = await AWSTaskRunner.describeTasks(clusterName, taskArn);
|
||||
({ timestamp, shouldReadLogs } = AWSTaskRunner.checkStreamingShouldContinue(taskData, timestamp, shouldReadLogs));
|
||||
({ iterator, shouldReadLogs, output, shouldCleanup } = await AWSTaskRunner.handleLogStreamIteration(
|
||||
kinesis,
|
||||
iterator,
|
||||
shouldReadLogs,
|
||||
taskDef,
|
||||
output,
|
||||
shouldCleanup,
|
||||
));
|
||||
@@ -150,23 +147,18 @@ class AWSTaskRunner {
|
||||
}
|
||||
|
||||
private static async handleLogStreamIteration(
|
||||
kinesis: AWS.Kinesis,
|
||||
iterator: string,
|
||||
shouldReadLogs: boolean,
|
||||
taskDef: CloudRunnerAWSTaskDef,
|
||||
output: string,
|
||||
shouldCleanup: boolean,
|
||||
) {
|
||||
const records = await kinesis
|
||||
.getRecords({
|
||||
ShardIterator: iterator,
|
||||
})
|
||||
.promise();
|
||||
const records = await AWSTaskRunner.Kinesis.getRecords({
|
||||
ShardIterator: iterator,
|
||||
}).promise();
|
||||
iterator = records.NextShardIterator || '';
|
||||
({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords(
|
||||
records,
|
||||
iterator,
|
||||
taskDef,
|
||||
shouldReadLogs,
|
||||
output,
|
||||
shouldCleanup,
|
||||
@@ -197,7 +189,6 @@ class AWSTaskRunner {
|
||||
private static logRecords(
|
||||
records,
|
||||
iterator: string,
|
||||
taskDef: CloudRunnerAWSTaskDef,
|
||||
shouldReadLogs: boolean,
|
||||
output: string,
|
||||
shouldCleanup: boolean,
|
||||
@@ -224,24 +215,20 @@ class AWSTaskRunner {
|
||||
return { shouldReadLogs, output, shouldCleanup };
|
||||
}
|
||||
|
||||
private static async getLogStream(kinesis: AWS.Kinesis, kinesisStreamName: string) {
|
||||
return await kinesis
|
||||
.describeStream({
|
||||
StreamName: kinesisStreamName,
|
||||
})
|
||||
.promise();
|
||||
private static async getLogStream(kinesisStreamName: string) {
|
||||
return await AWSTaskRunner.Kinesis.describeStream({
|
||||
StreamName: kinesisStreamName,
|
||||
}).promise();
|
||||
}
|
||||
|
||||
private static async getLogIterator(kinesis: AWS.Kinesis, stream) {
|
||||
private static async getLogIterator(stream) {
|
||||
return (
|
||||
(
|
||||
await kinesis
|
||||
.getShardIterator({
|
||||
ShardIteratorType: 'TRIM_HORIZON',
|
||||
StreamName: stream.StreamDescription.StreamName,
|
||||
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
||||
})
|
||||
.promise()
|
||||
await AWSTaskRunner.Kinesis.getShardIterator({
|
||||
ShardIteratorType: 'TRIM_HORIZON',
|
||||
StreamName: stream.StreamDescription.StreamName,
|
||||
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
||||
}).promise()
|
||||
).ShardIterator || ''
|
||||
);
|
||||
}
|
||||
|
||||
@@ -11,6 +11,10 @@ Parameters:
|
||||
Type: String
|
||||
Default: example
|
||||
Description: A name for the service
|
||||
LogGroupName:
|
||||
Type: String
|
||||
Default: example
|
||||
Description: Name to use for the log group created for this task
|
||||
ImageUrl:
|
||||
Type: String
|
||||
Default: nginx
|
||||
@@ -68,7 +72,7 @@ Resources:
|
||||
LogGroup:
|
||||
Type: 'AWS::Logs::LogGroup'
|
||||
Properties:
|
||||
LogGroupName: !Ref ServiceName
|
||||
LogGroupName: !Ref LogGroupName
|
||||
Metadata:
|
||||
'AWS::CloudFormation::Designer':
|
||||
id: aece53ae-b82d-4267-bc16-ed964b05db27
|
||||
@@ -78,7 +82,7 @@ Resources:
|
||||
FilterPattern: ''
|
||||
RoleArn:
|
||||
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:CloudWatchIAMRole'
|
||||
LogGroupName: !Ref ServiceName
|
||||
LogGroupName: !Ref LogGroupName
|
||||
DestinationArn:
|
||||
'Fn::GetAtt':
|
||||
- KinesisStream
|
||||
@@ -147,7 +151,7 @@ Resources:
|
||||
LogConfiguration:
|
||||
LogDriver: awslogs
|
||||
Options:
|
||||
awslogs-group: !Ref ServiceName
|
||||
awslogs-group: !Ref LogGroupName
|
||||
awslogs-region: !Ref 'AWS::Region'
|
||||
awslogs-stream-prefix: !Ref ServiceName
|
||||
DependsOn:
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
import AWS from 'aws-sdk';
|
||||
import { CliFunction } from '../../../../cli/cli-functions-repository';
|
||||
import Input from '../../../../input';
|
||||
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
|
||||
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
|
||||
|
||||
export class AwsCliCommands {
|
||||
@CliFunction(`aws-list-all`, `List all resources`)
|
||||
static async awsListAll() {
|
||||
await AwsCliCommands.awsListStacks(undefined, true);
|
||||
await AwsCliCommands.awsListTasks();
|
||||
await AwsCliCommands.awsListLogGroups(undefined, true);
|
||||
}
|
||||
@CliFunction(`aws-garbage-collect`, `garbage collect aws resources not in use !WIP!`)
|
||||
static async garbageCollectAws() {
|
||||
await AwsCliCommands.cleanup(false);
|
||||
}
|
||||
@CliFunction(`aws-garbage-collect-all`, `garbage collect aws resources regardless of whether they are in use`)
|
||||
static async garbageCollectAwsAll() {
|
||||
await AwsCliCommands.cleanup(true);
|
||||
}
|
||||
@CliFunction(
|
||||
`aws-garbage-collect-all-1d-older`,
|
||||
`garbage collect aws resources created more than 1d ago (ignore if they are in use)`,
|
||||
)
|
||||
static async garbageCollectAwsAllOlderThanOneDay() {
|
||||
await AwsCliCommands.cleanup(true, true);
|
||||
}
|
||||
static isOlderThan1day(date: any) {
|
||||
const ageDate = new Date(date.getTime() - Date.now());
|
||||
|
||||
return ageDate.getDay() > 0;
|
||||
}
|
||||
@CliFunction(`aws-list-stacks`, `List stacks`)
|
||||
static async awsListStacks(perResultCallback: any = false, verbose: boolean = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = new AWS.CloudFormation();
|
||||
const stacks =
|
||||
(await CF.listStacks().promise()).StackSummaries?.filter(
|
||||
(_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
|
||||
// _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
|
||||
) || [];
|
||||
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
|
||||
for (const element of stacks) {
|
||||
const ageDate = new Date(element.CreationTime.getTime() - Date.now());
|
||||
if (verbose)
|
||||
CloudRunnerLogger.log(
|
||||
`Task Stack ${element.StackName} - Age D${ageDate.getDay()} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
if (perResultCallback) await perResultCallback(element);
|
||||
}
|
||||
const baseStacks =
|
||||
(await CF.listStacks().promise()).StackSummaries?.filter(
|
||||
(_x) =>
|
||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
|
||||
) || [];
|
||||
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
|
||||
for (const element of baseStacks) {
|
||||
const ageDate = new Date(element.CreationTime.getTime() - Date.now());
|
||||
if (verbose)
|
||||
CloudRunnerLogger.log(
|
||||
`Base Stack ${
|
||||
element.StackName
|
||||
} - Age D${ageDate.getHours()} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
if (perResultCallback) await perResultCallback(element);
|
||||
}
|
||||
if (stacks === undefined) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@CliFunction(`aws-list-tasks`, `List tasks`)
|
||||
static async awsListTasks(perResultCallback: any = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const ecs = new AWS.ECS();
|
||||
const clusters = (await ecs.listClusters().promise()).clusterArns || [];
|
||||
CloudRunnerLogger.log(`Clusters ${clusters.length}`);
|
||||
for (const element of clusters) {
|
||||
const input: AWS.ECS.ListTasksRequest = {
|
||||
cluster: element,
|
||||
};
|
||||
|
||||
const list = (await ecs.listTasks(input).promise()).taskArns || [];
|
||||
if (list.length > 0) {
|
||||
const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
|
||||
const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
|
||||
if (describeList === []) {
|
||||
continue;
|
||||
}
|
||||
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
|
||||
for (const taskElement of describeList) {
|
||||
if (taskElement === undefined) {
|
||||
continue;
|
||||
}
|
||||
taskElement.overrides = {};
|
||||
taskElement.attachments = [];
|
||||
if (taskElement.createdAt === undefined) {
|
||||
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
|
||||
continue;
|
||||
}
|
||||
if (perResultCallback) await perResultCallback(taskElement, element);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@CliFunction(`aws-list-log-groups`, `List tasks`)
|
||||
static async awsListLogGroups(perResultCallback: any = false, verbose: boolean = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const ecs = new AWS.CloudWatchLogs();
|
||||
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
|
||||
/* logGroupNamePrefix: 'game-ci' */
|
||||
};
|
||||
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
|
||||
const logGroups = logGroupsDescribe.logGroups || [];
|
||||
while (logGroupsDescribe.nextToken) {
|
||||
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
|
||||
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
|
||||
logGroups.push(...(logGroupsDescribe?.logGroups || []));
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
|
||||
for (const element of logGroups) {
|
||||
if (element.creationTime === undefined) {
|
||||
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
|
||||
continue;
|
||||
}
|
||||
const ageDate = new Date(new Date(element.creationTime).getTime() - Date.now());
|
||||
if (verbose)
|
||||
CloudRunnerLogger.log(
|
||||
`Log Group Name ${
|
||||
element.logGroupName
|
||||
} - Age D${ageDate.getDay()} H${ageDate.getHours()} M${ageDate.getMinutes()} - 1d old ${AwsCliCommands.isOlderThan1day(
|
||||
new Date(element.creationTime),
|
||||
)}`,
|
||||
);
|
||||
if (perResultCallback) await perResultCallback(element, element);
|
||||
}
|
||||
}
|
||||
|
||||
private static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = new AWS.CloudFormation();
|
||||
const ecs = new AWS.ECS();
|
||||
const cwl = new AWS.CloudWatchLogs();
|
||||
await AwsCliCommands.awsListStacks(async (element) => {
|
||||
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(element.CreationTime))) {
|
||||
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
|
||||
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
|
||||
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
|
||||
const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
|
||||
await CF.deleteStack(deleteStackInput).promise();
|
||||
}
|
||||
});
|
||||
await AwsCliCommands.awsListTasks(async (taskElement, element) => {
|
||||
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(taskElement.CreatedAt))) {
|
||||
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
|
||||
await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
|
||||
}
|
||||
});
|
||||
await AwsCliCommands.awsListLogGroups(async (element) => {
|
||||
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(new Date(element.createdAt)))) {
|
||||
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
|
||||
await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,18 @@ import * as SDK from 'aws-sdk';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||
import AWSTaskRunner from './aws-task-runner';
|
||||
import AwsTaskRunner from './aws-task-runner';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { AWSJobStack } from './aws-job-stack';
|
||||
import { AWSBaseStack } from './aws-base-stack';
|
||||
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
|
||||
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
|
||||
import { Input } from '../../..';
|
||||
import { TertiaryResourcesService } from './services/tertiary-resources-service';
|
||||
import { GarbageCollectionService } from './services/garbage-collection-service';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { TaskService } from './services/task-service';
|
||||
|
||||
class AWSBuildEnvironment implements ProviderInterface {
|
||||
private baseStackName: string;
|
||||
@@ -16,7 +21,43 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
this.baseStackName = buildParameters.awsBaseStackName;
|
||||
}
|
||||
async cleanup(
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
await TaskService.awsListJobs();
|
||||
await TertiaryResourcesService.awsListLogGroups();
|
||||
await TaskService.awsListTasks();
|
||||
await TaskService.awsListStacks();
|
||||
|
||||
return [];
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
async watchWorkflow(): Promise<string> {
|
||||
return await TaskService.watch();
|
||||
}
|
||||
|
||||
async listOtherResources(): Promise<string> {
|
||||
await TertiaryResourcesService.awsListLogGroups();
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
async garbageCollect(
|
||||
filter: string,
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
await GarbageCollectionService.cleanup(!previewOnly);
|
||||
|
||||
return ``;
|
||||
}
|
||||
|
||||
async cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -26,7 +67,7 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
async setup(
|
||||
async setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -37,7 +78,7 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
|
||||
async runTask(
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
@@ -49,12 +90,14 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const ECS = new SDK.ECS();
|
||||
const CF = new SDK.CloudFormation();
|
||||
AwsTaskRunner.ECS = ECS;
|
||||
AwsTaskRunner.Kinesis = new SDK.Kinesis();
|
||||
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
|
||||
const entrypoint = ['/bin/sh'];
|
||||
const startTimeMs = Date.now();
|
||||
|
||||
await new AWSBaseStack(this.baseStackName).setupBaseStack(CF);
|
||||
const taskDef = await new AWSJobStack(this.baseStackName).setupCloudFormations(
|
||||
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
|
||||
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
|
||||
CF,
|
||||
buildGuid,
|
||||
image,
|
||||
@@ -69,7 +112,7 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
try {
|
||||
const postSetupStacksTimeMs = Date.now();
|
||||
CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
|
||||
const { output, shouldCleanup } = await AWSTaskRunner.runTask(taskDef, ECS, CF, environment, buildGuid, commands);
|
||||
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
|
||||
postRunTaskTimeMs = Date.now();
|
||||
CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
|
||||
if (shouldCleanup) {
|
||||
@@ -81,6 +124,7 @@ class AWSBuildEnvironment implements ProviderInterface {
|
||||
|
||||
return output;
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.log(`error running task ${error}`);
|
||||
await this.cleanupResources(CF, taskDef);
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import AWS from 'aws-sdk';
|
||||
import Input from '../../../../input';
|
||||
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
|
||||
import { TaskService } from './task-service';
|
||||
import { TertiaryResourcesService } from './tertiary-resources-service';
|
||||
|
||||
export class GarbageCollectionService {
|
||||
static isOlderThan1day(date: any) {
|
||||
const ageDate = new Date(date.getTime() - Date.now());
|
||||
|
||||
return ageDate.getDay() > 0;
|
||||
}
|
||||
|
||||
public static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = new AWS.CloudFormation();
|
||||
const ecs = new AWS.ECS();
|
||||
const cwl = new AWS.CloudWatchLogs();
|
||||
const taskDefinitionsInUse = new Array();
|
||||
await TaskService.awsListTasks(async (taskElement, element) => {
|
||||
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
|
||||
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
|
||||
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
|
||||
await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
|
||||
}
|
||||
});
|
||||
await TaskService.awsListStacks(async (element) => {
|
||||
if (
|
||||
(await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some(
|
||||
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
|
||||
)
|
||||
) {
|
||||
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
|
||||
|
||||
return;
|
||||
}
|
||||
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(element.CreationTime))) {
|
||||
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
|
||||
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
|
||||
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
|
||||
const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
|
||||
await CF.deleteStack(deleteStackInput).promise();
|
||||
}
|
||||
});
|
||||
await TertiaryResourcesService.awsListLogGroups(async (element) => {
|
||||
if (
|
||||
deleteResources &&
|
||||
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))
|
||||
) {
|
||||
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
|
||||
await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
131
src/model/cloud-runner/providers/aws/services/task-service.ts
Normal file
131
src/model/cloud-runner/providers/aws/services/task-service.ts
Normal file
@@ -0,0 +1,131 @@
|
||||
import AWS from 'aws-sdk';
|
||||
import Input from '../../../../input';
|
||||
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
|
||||
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
|
||||
import AwsTaskRunner from '../aws-task-runner';
|
||||
|
||||
export class TaskService {
|
||||
static async watch() {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
const { output, shouldCleanup } = await AwsTaskRunner.streamLogsUntilTaskStops(
|
||||
process.env.cluster || ``,
|
||||
process.env.taskArn || ``,
|
||||
process.env.streamName || ``,
|
||||
);
|
||||
|
||||
return output;
|
||||
}
|
||||
public static async awsListStacks(perResultCallback: any = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = new AWS.CloudFormation();
|
||||
const stacks =
|
||||
(await CF.listStacks().promise()).StackSummaries?.filter(
|
||||
(_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
|
||||
// _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
|
||||
) || [];
|
||||
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
|
||||
for (const element of stacks) {
|
||||
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
if (perResultCallback) await perResultCallback(element);
|
||||
}
|
||||
const baseStacks =
|
||||
(await CF.listStacks().promise()).StackSummaries?.filter(
|
||||
(_x) =>
|
||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
|
||||
) || [];
|
||||
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
|
||||
for (const element of baseStacks) {
|
||||
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
if (perResultCallback) await perResultCallback(element);
|
||||
}
|
||||
if (stacks === undefined) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
public static async awsListTasks(perResultCallback: any = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const ecs = new AWS.ECS();
|
||||
const clusters = (await ecs.listClusters().promise()).clusterArns || [];
|
||||
CloudRunnerLogger.log(`Clusters ${clusters.length}`);
|
||||
for (const element of clusters) {
|
||||
const input: AWS.ECS.ListTasksRequest = {
|
||||
cluster: element,
|
||||
};
|
||||
|
||||
const list = (await ecs.listTasks(input).promise()).taskArns || [];
|
||||
if (list.length > 0) {
|
||||
const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
|
||||
const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
|
||||
if (describeList.length === 0) {
|
||||
continue;
|
||||
}
|
||||
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
|
||||
for (const taskElement of describeList) {
|
||||
if (taskElement === undefined) {
|
||||
continue;
|
||||
}
|
||||
taskElement.overrides = {};
|
||||
taskElement.attachments = [];
|
||||
if (taskElement.createdAt === undefined) {
|
||||
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
|
||||
continue;
|
||||
}
|
||||
if (perResultCallback) await perResultCallback(taskElement, element);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
public static async awsListJobs(perResultCallback: any = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = new AWS.CloudFormation();
|
||||
const stacks =
|
||||
(await CF.listStacks().promise()).StackSummaries?.filter(
|
||||
(_x) =>
|
||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
|
||||
) || [];
|
||||
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
|
||||
for (const element of stacks) {
|
||||
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
if (perResultCallback) await perResultCallback(element);
|
||||
}
|
||||
}
|
||||
public static async awsDescribeJob(job: string) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = new AWS.CloudFormation();
|
||||
const stack = (await CF.listStacks().promise()).StackSummaries?.find((_x) => _x.StackName === job) || undefined;
|
||||
const stackInfo = (await CF.describeStackResources({ StackName: job }).promise()) || undefined;
|
||||
const stackInfo2 = (await CF.describeStacks({ StackName: job }).promise()) || undefined;
|
||||
if (stack === undefined) {
|
||||
throw new Error('stack not defined');
|
||||
}
|
||||
const ageDate: Date = new Date(Date.now() - stack.CreationTime.getTime());
|
||||
const message = `
|
||||
Task Stack ${stack.StackName}
|
||||
Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}
|
||||
${JSON.stringify(stack, undefined, 4)}
|
||||
${JSON.stringify(stackInfo, undefined, 4)}
|
||||
${JSON.stringify(stackInfo2, undefined, 4)}
|
||||
`;
|
||||
CloudRunnerLogger.log(message);
|
||||
|
||||
return message;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import AWS from 'aws-sdk';
|
||||
import Input from '../../../../input';
|
||||
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
|
||||
|
||||
export class TertiaryResourcesService {
|
||||
public static async awsListLogGroups(perResultCallback: any = false) {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const ecs = new AWS.CloudWatchLogs();
|
||||
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
|
||||
/* logGroupNamePrefix: 'game-ci' */
|
||||
};
|
||||
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
|
||||
const logGroups = logGroupsDescribe.logGroups || [];
|
||||
while (logGroupsDescribe.nextToken) {
|
||||
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
|
||||
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
|
||||
logGroups.push(...(logGroupsDescribe?.logGroups || []));
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
|
||||
for (const element of logGroups) {
|
||||
if (element.creationTime === undefined) {
|
||||
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
|
||||
continue;
|
||||
}
|
||||
const ageDate: Date = new Date(Date.now() - element.creationTime);
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
`Task Stack ${element.logGroupName} - Age D${Math.floor(
|
||||
ageDate.getHours() / 24,
|
||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
||||
);
|
||||
if (perResultCallback) await perResultCallback(element, element);
|
||||
}
|
||||
}
|
||||
}
|
||||
148
src/model/cloud-runner/providers/docker/index.ts
Normal file
148
src/model/cloud-runner/providers/docker/index.ts
Normal file
@@ -0,0 +1,148 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import Docker from '../../../docker';
|
||||
import { Action } from '../../..';
|
||||
import { writeFileSync } from 'fs';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
|
||||
import * as fs from 'fs';
|
||||
|
||||
class LocalDockerCloudRunner implements ProviderInterface {
|
||||
public buildParameters: BuildParameters | undefined;
|
||||
|
||||
listResources(): Promise<ProviderResource[]> {
|
||||
return new Promise((resolve) => resolve([]));
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
return new Promise((result) => result(``));
|
||||
}
|
||||
async cleanupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
const { workspace } = Action;
|
||||
if (fs.existsSync(`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar.lz4`)) {
|
||||
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar.lz4`,
|
||||
);
|
||||
}
|
||||
}
|
||||
setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
this.buildParameters = buildParameters;
|
||||
}
|
||||
|
||||
public async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string> {
|
||||
CloudRunnerLogger.log(buildGuid);
|
||||
CloudRunnerLogger.log(commands);
|
||||
|
||||
const { workspace, actionFolder } = Action;
|
||||
const content: any[] = [];
|
||||
for (const x of secrets) {
|
||||
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
|
||||
}
|
||||
for (const x of environment) {
|
||||
content.push({ name: x.name, value: x.value });
|
||||
}
|
||||
|
||||
// if (this.buildParameters?.cloudRunnerIntegrationTests) {
|
||||
// core.info(JSON.stringify(content, undefined, 4));
|
||||
// core.info(JSON.stringify(secrets, undefined, 4));
|
||||
// core.info(JSON.stringify(environment, undefined, 4));
|
||||
// }
|
||||
|
||||
// eslint-disable-next-line unicorn/no-for-loop
|
||||
for (let index = 0; index < content.length; index++) {
|
||||
if (content[index] === undefined) {
|
||||
delete content[index];
|
||||
}
|
||||
}
|
||||
let myOutput = '';
|
||||
const sharedFolder = `/data/`;
|
||||
|
||||
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
|
||||
const entrypointFilePath = `start.sh`;
|
||||
const fileContents = `#!/bin/bash
|
||||
set -e
|
||||
|
||||
mkdir -p /github/workspace/cloud-runner-cache
|
||||
mkdir -p /data/cache
|
||||
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
|
||||
${commands}
|
||||
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
|
||||
`;
|
||||
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
|
||||
flag: 'w',
|
||||
});
|
||||
|
||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
|
||||
}
|
||||
|
||||
if (fs.existsSync(`${workspace}/cloud-runner-cache`)) {
|
||||
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache && du -sh ${workspace}/cloud-runner-cache`);
|
||||
}
|
||||
await Docker.run(
|
||||
image,
|
||||
{ workspace, actionFolder, ...this.buildParameters },
|
||||
false,
|
||||
`chmod +x /github/workspace/${entrypointFilePath} && /github/workspace/${entrypointFilePath}`,
|
||||
content,
|
||||
{
|
||||
listeners: {
|
||||
stdout: (data: Buffer) => {
|
||||
myOutput += data.toString();
|
||||
},
|
||||
stderr: (data: Buffer) => {
|
||||
myOutput += `[LOCAL-DOCKER-ERROR]${data.toString()}`;
|
||||
},
|
||||
},
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
||||
return myOutput;
|
||||
}
|
||||
}
|
||||
export default LocalDockerCloudRunner;
|
||||
@@ -1,5 +1,5 @@
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import { BuildParameters, Output } from '../../..';
|
||||
import { BuildParameters } from '../../..';
|
||||
import * as core from '@actions/core';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
@@ -7,39 +7,84 @@ import KubernetesStorage from './kubernetes-storage';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import KubernetesTaskRunner from './kubernetes-task-runner';
|
||||
import KubernetesSecret from './kubernetes-secret';
|
||||
import waitUntil from 'async-wait-until';
|
||||
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
|
||||
import KubernetesServiceAccount from './kubernetes-service-account';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
import DependencyOverrideService from '../../services/depdency-override-service';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
import KubernetesPods from './kubernetes-pods';
|
||||
|
||||
class Kubernetes implements ProviderInterface {
|
||||
private kubeConfig: k8s.KubeConfig;
|
||||
private kubeClient: k8s.CoreV1Api;
|
||||
private kubeClientBatch: k8s.BatchV1Api;
|
||||
private buildGuid: string = '';
|
||||
private buildParameters: BuildParameters;
|
||||
private pvcName: string = '';
|
||||
private secretName: string = '';
|
||||
private jobName: string = '';
|
||||
private namespace: string;
|
||||
private podName: string = '';
|
||||
private containerName: string = '';
|
||||
private cleanupCronJobName: string = '';
|
||||
private serviceAccountName: string = '';
|
||||
public static Instance: Kubernetes;
|
||||
public kubeConfig!: k8s.KubeConfig;
|
||||
public kubeClient!: k8s.CoreV1Api;
|
||||
public kubeClientBatch!: k8s.BatchV1Api;
|
||||
public buildGuid: string = '';
|
||||
public buildParameters!: BuildParameters;
|
||||
public pvcName: string = '';
|
||||
public secretName: string = '';
|
||||
public jobName: string = '';
|
||||
public namespace!: string;
|
||||
public podName: string = '';
|
||||
public containerName: string = '';
|
||||
public cleanupCronJobName: string = '';
|
||||
public serviceAccountName: string = '';
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
Kubernetes.Instance = this;
|
||||
this.kubeConfig = new k8s.KubeConfig();
|
||||
this.kubeConfig.loadFromDefault();
|
||||
this.kubeClient = this.kubeConfig.makeApiClient(k8s.CoreV1Api);
|
||||
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
|
||||
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment');
|
||||
|
||||
this.namespace = 'default';
|
||||
this.buildParameters = buildParameters;
|
||||
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment');
|
||||
}
|
||||
public async setup(
|
||||
|
||||
async listResources(): Promise<ProviderResource[]> {
|
||||
const pods = await this.kubeClient.listNamespacedPod(this.namespace);
|
||||
const serviceAccounts = await this.kubeClient.listNamespacedServiceAccount(this.namespace);
|
||||
const secrets = await this.kubeClient.listNamespacedSecret(this.namespace);
|
||||
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
|
||||
|
||||
return [
|
||||
...pods.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...serviceAccounts.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...secrets.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
...jobs.body.items.map((x) => {
|
||||
return { Name: x.metadata?.name || `` };
|
||||
}),
|
||||
];
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
return new Promise((result) => result(``));
|
||||
}
|
||||
public async setupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -48,12 +93,11 @@ class Kubernetes implements ProviderInterface {
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
try {
|
||||
this.pvcName = `unity-builder-pvc-${buildGuid}`;
|
||||
this.cleanupCronJobName = `unity-builder-cronjob-${buildGuid}`;
|
||||
this.serviceAccountName = `service-account-${buildGuid}`;
|
||||
if (await DependencyOverrideService.CheckHealth()) {
|
||||
await DependencyOverrideService.TryStartDependencies();
|
||||
}
|
||||
this.buildParameters = buildParameters;
|
||||
const id = buildParameters.retainWorkspace ? CloudRunner.lockedWorkspace : buildParameters.buildGuid;
|
||||
this.pvcName = `unity-builder-pvc-${id}`;
|
||||
this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
|
||||
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
|
||||
await KubernetesStorage.createPersistentVolumeClaim(
|
||||
buildParameters,
|
||||
this.pvcName,
|
||||
@@ -67,7 +111,7 @@ class Kubernetes implements ProviderInterface {
|
||||
}
|
||||
}
|
||||
|
||||
async runTask(
|
||||
async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
@@ -77,40 +121,22 @@ class Kubernetes implements ProviderInterface {
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string> {
|
||||
try {
|
||||
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
|
||||
|
||||
// Setup
|
||||
this.buildGuid = buildGuid;
|
||||
this.secretName = `build-credentials-${buildGuid}`;
|
||||
this.jobName = `unity-builder-job-${buildGuid}`;
|
||||
this.secretName = `build-credentials-${this.buildGuid}`;
|
||||
this.jobName = `unity-builder-job-${this.buildGuid}`;
|
||||
this.containerName = `main`;
|
||||
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
|
||||
const jobSpec = KubernetesJobSpecFactory.getJobSpec(
|
||||
commands,
|
||||
image,
|
||||
mountdir,
|
||||
workingdir,
|
||||
environment,
|
||||
secrets,
|
||||
this.buildGuid,
|
||||
this.buildParameters,
|
||||
this.secretName,
|
||||
this.pvcName,
|
||||
this.jobName,
|
||||
k8s,
|
||||
);
|
||||
|
||||
// Run
|
||||
const jobResult = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
|
||||
CloudRunnerLogger.log(`Creating build job ${JSON.stringify(jobResult.body.metadata, undefined, 4)}`);
|
||||
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
CloudRunnerLogger.log('Job created');
|
||||
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
|
||||
this.setPodNameAndContainerName(await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
|
||||
CloudRunnerLogger.log('Watching pod until running');
|
||||
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
|
||||
let output = '';
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
try {
|
||||
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
|
||||
CloudRunnerLogger.log('Pod running, streaming logs');
|
||||
output = await KubernetesTaskRunner.runTask(
|
||||
this.kubeConfig,
|
||||
@@ -120,11 +146,25 @@ class Kubernetes implements ProviderInterface {
|
||||
'main',
|
||||
this.namespace,
|
||||
);
|
||||
break;
|
||||
const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient);
|
||||
|
||||
if (!running) {
|
||||
CloudRunnerLogger.log(`Pod not found, assumed ended!`);
|
||||
break;
|
||||
} else {
|
||||
CloudRunnerLogger.log('Pod still running, recovering stream...');
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.message.includes(`HTTP`)) {
|
||||
const reason = error.response?.body?.reason || error.reason || ``;
|
||||
const errorMessage = error.message || ``;
|
||||
|
||||
const continueStreaming = reason === `NotFound` || errorMessage.includes(`dial timeout, backstop`);
|
||||
if (continueStreaming) {
|
||||
CloudRunnerLogger.log('Log Stream Container Not Found');
|
||||
await new Promise((resolve) => resolve(5000));
|
||||
continue;
|
||||
} else {
|
||||
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -140,6 +180,44 @@ class Kubernetes implements ProviderInterface {
|
||||
}
|
||||
}
|
||||
|
||||
private async createNamespacedJob(
|
||||
commands: string,
|
||||
image: string,
|
||||
mountdir: string,
|
||||
workingdir: string,
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
) {
|
||||
for (let index = 0; index < 3; index++) {
|
||||
try {
|
||||
const jobSpec = KubernetesJobSpecFactory.getJobSpec(
|
||||
commands,
|
||||
image,
|
||||
mountdir,
|
||||
workingdir,
|
||||
environment,
|
||||
secrets,
|
||||
this.buildGuid,
|
||||
this.buildParameters,
|
||||
this.secretName,
|
||||
this.pvcName,
|
||||
this.jobName,
|
||||
k8s,
|
||||
);
|
||||
await new Promise((promise) => setTimeout(promise, 15000));
|
||||
await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
|
||||
CloudRunnerLogger.log(`Build job created`);
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
CloudRunnerLogger.log('Job created');
|
||||
|
||||
return;
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.log(`Error occured creating job: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setPodNameAndContainerName(pod: k8s.V1Pod) {
|
||||
this.podName = pod.metadata?.name || '';
|
||||
this.containerName = pod.status?.containerStatuses?.[0].name || '';
|
||||
@@ -150,32 +228,24 @@ class Kubernetes implements ProviderInterface {
|
||||
try {
|
||||
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
||||
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
|
||||
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
||||
await new Promise((promise) => setTimeout(promise, 5000));
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.log('Failed to cleanup, error:');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
CloudRunnerLogger.log('Abandoning cleanup, build error:');
|
||||
throw error;
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`Failed to cleanup`);
|
||||
if (error.response.body.reason !== `NotFound`) {
|
||||
CloudRunnerLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await waitUntil(
|
||||
async () => {
|
||||
const jobBody = (await this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body;
|
||||
const podBody = (await this.kubeClient.readNamespacedPod(this.podName, this.namespace)).body;
|
||||
|
||||
return (jobBody === null || jobBody.status?.active === 0) && podBody === null;
|
||||
},
|
||||
{
|
||||
timeout: 500000,
|
||||
intervalBetweenAttempts: 15000,
|
||||
},
|
||||
);
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch {}
|
||||
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`Failed to cleanup secret`);
|
||||
CloudRunnerLogger.log(error.response.body.reason);
|
||||
}
|
||||
CloudRunnerLogger.log('cleaned up Secret, Job and Pod');
|
||||
CloudRunnerLogger.log('cleaning up finished');
|
||||
}
|
||||
|
||||
async cleanup(
|
||||
async cleanupWorkflow(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -183,11 +253,19 @@ class Kubernetes implements ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {
|
||||
if (buildParameters.retainWorkspace) {
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(`deleting PVC`);
|
||||
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
|
||||
await Output.setBuildVersion(buildParameters.buildVersion);
|
||||
// eslint-disable-next-line unicorn/no-process-exit
|
||||
process.exit();
|
||||
|
||||
try {
|
||||
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
|
||||
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
|
||||
CloudRunnerLogger.log('cleaned up PVC and Service Account');
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
static async findPodFromJob(kubeClient: CoreV1Api, jobName: string, namespace: string) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../../services/cloud-runner-build-command-process';
|
||||
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
@@ -103,7 +103,7 @@ class KubernetesJobSpecFactory {
|
||||
name: 'main',
|
||||
image,
|
||||
command: ['/bin/sh'],
|
||||
args: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(command, CloudRunner.buildParameters)],
|
||||
args: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(command, CloudRunner.buildParameters)],
|
||||
|
||||
workingDir: `${workingDirectory}`,
|
||||
resources: {
|
||||
|
||||
14
src/model/cloud-runner/providers/k8s/kubernetes-pods.ts
Normal file
14
src/model/cloud-runner/providers/k8s/kubernetes-pods.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
class KubernetesPods {
|
||||
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
||||
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
|
||||
const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
|
||||
const phase = pods[0].status?.phase || 'undefined status';
|
||||
CloudRunnerLogger.log(`Getting pod status: ${phase}`);
|
||||
|
||||
return running;
|
||||
}
|
||||
}
|
||||
|
||||
export default KubernetesPods;
|
||||
@@ -1,6 +1,7 @@
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
const base64 = require('base-64');
|
||||
|
||||
class KubernetesSecret {
|
||||
@@ -10,19 +11,34 @@ class KubernetesSecret {
|
||||
namespace: string,
|
||||
kubeClient: CoreV1Api,
|
||||
) {
|
||||
const secret = new k8s.V1Secret();
|
||||
secret.apiVersion = 'v1';
|
||||
secret.kind = 'Secret';
|
||||
secret.type = 'Opaque';
|
||||
secret.metadata = {
|
||||
name: secretName,
|
||||
};
|
||||
secret.data = {};
|
||||
for (const buildSecret of secrets) {
|
||||
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
|
||||
}
|
||||
try {
|
||||
const secret = new k8s.V1Secret();
|
||||
secret.apiVersion = 'v1';
|
||||
secret.kind = 'Secret';
|
||||
secret.type = 'Opaque';
|
||||
secret.metadata = {
|
||||
name: secretName,
|
||||
};
|
||||
secret.data = {};
|
||||
for (const buildSecret of secrets) {
|
||||
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
|
||||
}
|
||||
CloudRunnerLogger.log(`Creating secret: ${secretName}`);
|
||||
const existingSecrets = await kubeClient.listNamespacedSecret(namespace);
|
||||
const mappedSecrets = existingSecrets.body.items.map((x) => {
|
||||
return x.metadata?.name || `no name`;
|
||||
});
|
||||
|
||||
return kubeClient.createNamespacedSecret(namespace, secret);
|
||||
CloudRunnerLogger.log(
|
||||
`ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`,
|
||||
);
|
||||
await new Promise((promise) => setTimeout(promise, 15000));
|
||||
await kubeClient.createNamespacedSecret(namespace, secret);
|
||||
CloudRunnerLogger.log('Created secret');
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.log(`Created secret failed ${error}`);
|
||||
throw new Error(`Failed to create kubernetes secret`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,8 @@ import * as core from '@actions/core';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import YAML from 'yaml';
|
||||
import { IncomingMessage } from 'http';
|
||||
import GitHub from '../../../github';
|
||||
|
||||
class KubernetesStorage {
|
||||
public static async createPersistentVolumeClaim(
|
||||
@@ -13,20 +13,19 @@ class KubernetesStorage {
|
||||
kubeClient: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
) {
|
||||
if (buildParameters.kubeVolume) {
|
||||
CloudRunnerLogger.log(buildParameters.kubeVolume);
|
||||
if (buildParameters.kubeVolume !== ``) {
|
||||
CloudRunnerLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`);
|
||||
pvcName = buildParameters.kubeVolume;
|
||||
|
||||
return;
|
||||
}
|
||||
const pvcList = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items.map(
|
||||
(x) => x.metadata?.name,
|
||||
);
|
||||
const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items;
|
||||
const pvcList = allPvc.map((x) => x.metadata?.name);
|
||||
CloudRunnerLogger.log(`Current PVCs in namespace ${namespace}`);
|
||||
CloudRunnerLogger.log(JSON.stringify(pvcList, undefined, 4));
|
||||
if (pvcList.includes(pvcName)) {
|
||||
CloudRunnerLogger.log(`pvc ${pvcName} already exists`);
|
||||
if (!buildParameters.isCliMode) {
|
||||
if (GitHub.githubInputEnabled) {
|
||||
core.setOutput('volume', pvcName);
|
||||
}
|
||||
|
||||
@@ -95,9 +94,6 @@ class KubernetesStorage {
|
||||
},
|
||||
},
|
||||
};
|
||||
if (process.env.K8s_STORAGE_PVC_SPEC) {
|
||||
YAML.parse(process.env.K8s_STORAGE_PVC_SPEC);
|
||||
}
|
||||
const result = await kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
||||
|
||||
return result;
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
|
||||
class LocalDockerCloudRunner implements ProviderInterface {
|
||||
cleanup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
setup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public runTask(
|
||||
commands: string,
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
image: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
mountdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
workingdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string> {
|
||||
CloudRunnerLogger.log(buildGuid);
|
||||
CloudRunnerLogger.log(commands);
|
||||
|
||||
return CloudRunnerSystem.Run(commands, false, false);
|
||||
}
|
||||
}
|
||||
export default LocalDockerCloudRunner;
|
||||
@@ -4,9 +4,34 @@ import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environm
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
|
||||
class LocalCloudRunner implements ProviderInterface {
|
||||
cleanup(
|
||||
listResources(): Promise<ProviderResource[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -16,7 +41,7 @@ class LocalCloudRunner implements ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public setup(
|
||||
public setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -26,7 +51,7 @@ class LocalCloudRunner implements ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public async runTask(
|
||||
public async runTaskInWorkflow(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import { ProviderResource } from './provider-resource';
|
||||
import { ProviderWorkflow } from './provider-workflow';
|
||||
|
||||
export interface ProviderInterface {
|
||||
cleanup(
|
||||
cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -13,7 +15,7 @@ export interface ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
);
|
||||
setup(
|
||||
setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -23,7 +25,7 @@ export interface ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
);
|
||||
runTask(
|
||||
runTaskInWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -39,4 +41,19 @@ export interface ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string>;
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
olderThan: Number,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
fullCache: boolean,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
baseDependencies: boolean,
|
||||
): Promise<string>;
|
||||
listResources(): Promise<ProviderResource[]>;
|
||||
listWorkflow(): Promise<ProviderWorkflow[]>;
|
||||
watchWorkflow(): Promise<string>;
|
||||
}
|
||||
|
||||
3
src/model/cloud-runner/providers/provider-resource.ts
Normal file
3
src/model/cloud-runner/providers/provider-resource.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export class ProviderResource {
|
||||
public Name!: string;
|
||||
}
|
||||
3
src/model/cloud-runner/providers/provider-workflow.ts
Normal file
3
src/model/cloud-runner/providers/provider-workflow.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export class ProviderWorkflow {
|
||||
public Name!: string;
|
||||
}
|
||||
@@ -3,9 +3,28 @@ import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environm
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import { ProviderResource } from '../provider-resource';
|
||||
import { ProviderWorkflow } from '../provider-workflow';
|
||||
|
||||
class TestCloudRunner implements ProviderInterface {
|
||||
cleanup(
|
||||
listResources(): Promise<ProviderResource[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
watchWorkflow(): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
garbageCollect(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
filter: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
previewOnly: boolean,
|
||||
): Promise<string> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
cleanupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -15,7 +34,7 @@ class TestCloudRunner implements ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
setup(
|
||||
setupWorkflow(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@@ -25,7 +44,7 @@ class TestCloudRunner implements ProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public async runTask(
|
||||
public async runTaskInWorkflow(
|
||||
commands: string,
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
|
||||
Reference in New Issue
Block a user