Cloud runner develop v0.1 (#395)

* Correct aws logs link * Correct aws logs link * better aws cli commands and better cleanup for aws * better aws cli commands and better cleanup for aws * improved garbage collection cli options * Only allow ephemeral runners when using cloud runner integration tests flag to avoid unexpected hangup * Only allow ephemeral runners when using cloud runner integration tests flag to avoid unexpected hangup * fix issue #393 * Extract follow log stream service * consolidate into one pipeline file * consolidate into one pipeline file
2022-05-05 00:25:17 +01:00
parent 4556fc4ff1
commit f77696efae
13 changed files with 469 additions and 290 deletions
--- a/src/model/cloud-runner/providers/aws/aws-task-runner.ts
+++ b/src/model/cloud-runner/providers/aws/aws-task-runner.ts
@@ -6,8 +6,8 @@ import * as zlib from 'zlib';
 import CloudRunnerLogger from '../../services/cloud-runner-logger';
 import { Input } from '../../..';
 import CloudRunner from '../../cloud-runner';
-import { CloudRunnerStatics } from '../../cloud-runner-statics';
 import { CloudRunnerBuildCommandProcessor } from '../../services/cloud-runner-build-command-process';
+import { FollowLogStreamService } from '../../services/follow-log-stream-service';

 class AWSTaskRunner {
  static async runTask(
@@ -126,8 +126,8 @@ class AWSTaskRunner {
    const stream = await AWSTaskRunner.getLogStream(kinesis, kinesisStreamName);
    let iterator = await AWSTaskRunner.getLogIterator(kinesis, stream);

-    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${CF.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`;
-    CloudRunnerLogger.log(`You can also see the logs at AWS Cloud Watch: ${logBaseUrl}`);
+    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
+    CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
    let shouldReadLogs = true;
    let shouldCleanup = true;
    let timestamp: number = 0;
@@ -209,30 +209,13 @@ class AWSTaskRunner {
        );
        if (json.messageType === 'DATA_MESSAGE') {
          for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
-            let message = json.logEvents[logEventsIndex].message;
-            if (json.logEvents[logEventsIndex].message.includes(`---${CloudRunner.buildParameters.logId}`)) {
-              CloudRunnerLogger.log('End of log transmission received');
-              shouldReadLogs = false;
-            } else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
-              core.warning('LIBRARY NOT FOUND!');
-              core.setOutput('library-found', 'false');
-            } else if (message.includes('Build succeeded')) {
-              core.setOutput('build-result', 'success');
-            } else if (message.includes('Build fail')) {
-              core.setOutput('build-result', 'failed');
-              core.setFailed('unity build failed');
-              core.error('BUILD FAILED!');
-            } else if (message.includes(': Listening for Jobs')) {
-              core.setOutput('cloud runner stop watching', 'true');
-              shouldReadLogs = false;
-              shouldCleanup = false;
-              core.warning('cloud runner stop watching');
-            }
-            message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
-            if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
-              output += message;
-            }
-            CloudRunnerLogger.log(message);
+            const message = json.logEvents[logEventsIndex].message;
+            ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
+              message,
+              shouldReadLogs,
+              shouldCleanup,
+              output,
+            ));
          }
        }
      }
--- a/src/model/cloud-runner/providers/aws/cloud-formations/base-stack-formation.ts
+++ b/src/model/cloud-runner/providers/aws/cloud-formations/base-stack-formation.ts
@@ -1,6 +1,7 @@
 export class BaseStackFormation {
+  public static readonly baseStackDecription = `Game-CI base stack`;
  public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09'
-Description: Game-CI base stack
+Description: ${BaseStackFormation.baseStackDecription}
 Parameters:
  EnvironmentName:
    Type: String
--- a/src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts
+++ b/src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts
@@ -1,9 +1,7 @@
 export class TaskDefinitionFormation {
+  public static readonly description: string = `Game CI Cloud Runner Task Stack`;
  public static readonly formation: string = `AWSTemplateFormatVersion: 2010-09-09
-Description: >-
-  AWS Fargate cluster that can span public and private subnets. Supports public
-  facing load balancers, private internal load balancers, and both internal and
-  external service discovery namespaces.
+Description: ${TaskDefinitionFormation.description}
 Parameters:
  EnvironmentName:
    Type: String
--- a/src/model/cloud-runner/providers/aws/commands/aws-cli-commands.ts
+++ b/src/model/cloud-runner/providers/aws/commands/aws-cli-commands.ts
@@ -2,18 +2,67 @@ import AWS from 'aws-sdk';
 import { CliFunction } from '../../../../cli/cli-functions-repository';
 import Input from '../../../../input';
 import CloudRunnerLogger from '../../../services/cloud-runner-logger';
+import { BaseStackFormation } from '../cloud-formations/base-stack-formation';

 export class AwsCliCommands {
+  @CliFunction(`aws-list-all`, `List all resources`)
+  static async awsListAll() {
+    await AwsCliCommands.awsListStacks(undefined, true);
+    await AwsCliCommands.awsListTasks();
+    await AwsCliCommands.awsListLogGroups(undefined, true);
+  }
+  @CliFunction(`aws-garbage-collect`, `garbage collect aws resources not in use !WIP!`)
+  static async garbageCollectAws() {
+    await AwsCliCommands.cleanup(false);
+  }
+  @CliFunction(`aws-garbage-collect-all`, `garbage collect aws resources regardless of whether they are in use`)
+  static async garbageCollectAwsAll() {
+    await AwsCliCommands.cleanup(true);
+  }
+  @CliFunction(
+    `aws-garbage-collect-all-1d-older`,
+    `garbage collect aws resources created more than 1d ago (ignore if they are in use)`,
+  )
+  static async garbageCollectAwsAllOlderThanOneDay() {
+    await AwsCliCommands.cleanup(true, true);
+  }
+  static isOlderThan1day(date: any) {
+    const ageDate = new Date(date.getTime() - Date.now());
+
+    return ageDate.getDay() > 0;
+  }
  @CliFunction(`aws-list-stacks`, `List stacks`)
-  static async awsListStacks(perResultCallback: any) {
+  static async awsListStacks(perResultCallback: any = false, verbose: boolean = false) {
    process.env.AWS_REGION = Input.region;
    const CF = new AWS.CloudFormation();
    const stacks =
-      (await CF.listStacks().promise()).StackSummaries?.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE') || [];
-    CloudRunnerLogger.log(`DescribeStacksRequest ${stacks.length}`);
+      (await CF.listStacks().promise()).StackSummaries?.filter(
+        (_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
+        // _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
+      ) || [];
+    CloudRunnerLogger.log(`Stacks ${stacks.length}`);
    for (const element of stacks) {
-      CloudRunnerLogger.log(JSON.stringify(element, undefined, 4));
-      CloudRunnerLogger.log(`${element.StackName}`);
+      const ageDate = new Date(element.CreationTime.getTime() - Date.now());
+      if (verbose)
+        CloudRunnerLogger.log(
+          `Task Stack ${element.StackName} - Age D${ageDate.getDay()} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
+        );
+      if (perResultCallback) await perResultCallback(element);
+    }
+    const baseStacks =
+      (await CF.listStacks().promise()).StackSummaries?.filter(
+        (_x) =>
+          _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
+      ) || [];
+    CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
+    for (const element of baseStacks) {
+      const ageDate = new Date(element.CreationTime.getTime() - Date.now());
+      if (verbose)
+        CloudRunnerLogger.log(
+          `Base Stack ${
+            element.StackName
+          } - Age D${ageDate.getHours()} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
+        );
      if (perResultCallback) await perResultCallback(element);
    }
    if (stacks === undefined) {
@@ -21,15 +70,16 @@ export class AwsCliCommands {
    }
  }
  @CliFunction(`aws-list-tasks`, `List tasks`)
-  static async awsListTasks(perResultCallback: any) {
+  static async awsListTasks(perResultCallback: any = false) {
    process.env.AWS_REGION = Input.region;
-    CloudRunnerLogger.log(`ECS Clusters`);
    const ecs = new AWS.ECS();
    const clusters = (await ecs.listClusters().promise()).clusterArns || [];
+    CloudRunnerLogger.log(`Clusters ${clusters.length}`);
    for (const element of clusters) {
      const input: AWS.ECS.ListTasksRequest = {
        cluster: element,
      };
+
      const list = (await ecs.listTasks(input).promise()).taskArns || [];
      if (list.length > 0) {
        const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
@@ -37,14 +87,13 @@ export class AwsCliCommands {
        if (describeList === []) {
          continue;
        }
-        CloudRunnerLogger.log(`DescribeTasksRequest ${describeList.length}`);
+        CloudRunnerLogger.log(`Tasks ${describeList.length}`);
        for (const taskElement of describeList) {
          if (taskElement === undefined) {
            continue;
          }
          taskElement.overrides = {};
          taskElement.attachments = [];
-          CloudRunnerLogger.log(JSON.stringify(taskElement, undefined, 4));
          if (taskElement.createdAt === undefined) {
            CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
            continue;
@@ -54,39 +103,68 @@ export class AwsCliCommands {
      }
    }
  }
+  @CliFunction(`aws-list-log-groups`, `List tasks`)
+  static async awsListLogGroups(perResultCallback: any = false, verbose: boolean = false) {
+    process.env.AWS_REGION = Input.region;
+    const ecs = new AWS.CloudWatchLogs();
+    let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
+      /* logGroupNamePrefix: 'game-ci' */
+    };
+    let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
+    const logGroups = logGroupsDescribe.logGroups || [];
+    while (logGroupsDescribe.nextToken) {
+      logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
+      logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
+      logGroups.push(...(logGroupsDescribe?.logGroups || []));
+    }

-  @CliFunction(`aws-garbage-collect`, `garbage collect aws`)
-  static async garbageCollectAws() {
-    await AwsCliCommands.cleanup(false);
-  }
-  @CliFunction(`aws-garbage-collect-all`, `garbage collect aws`)
-  static async garbageCollectAwsAll() {
-    await AwsCliCommands.cleanup(true);
-  }
-  @CliFunction(`aws-garbage-collect-all-1d-older`, `garbage collect aws`)
-  static async garbageCollectAwsAllOlderThanOneDay() {
-    await AwsCliCommands.cleanup(true);
+    CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
+    for (const element of logGroups) {
+      if (element.creationTime === undefined) {
+        CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
+        continue;
+      }
+      const ageDate = new Date(new Date(element.creationTime).getTime() - Date.now());
+      if (verbose)
+        CloudRunnerLogger.log(
+          `Log Group Name ${
+            element.logGroupName
+          } - Age D${ageDate.getDay()} H${ageDate.getHours()} M${ageDate.getMinutes()} - 1d old ${AwsCliCommands.isOlderThan1day(
+            new Date(element.creationTime),
+          )}`,
+        );
+      if (perResultCallback) await perResultCallback(element, element);
+    }
  }

-  private static async cleanup(deleteResources = false) {
+  private static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
    process.env.AWS_REGION = Input.region;
    const CF = new AWS.CloudFormation();
    const ecs = new AWS.ECS();
+    const cwl = new AWS.CloudWatchLogs();
    await AwsCliCommands.awsListStacks(async (element) => {
-      if (deleteResources) {
+      if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(element.CreationTime))) {
        if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
          CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);

          return;
        }
+        CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
        const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
        await CF.deleteStack(deleteStackInput).promise();
      }
    });
    await AwsCliCommands.awsListTasks(async (taskElement, element) => {
-      if (deleteResources) {
+      if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(taskElement.CreatedAt))) {
+        CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
        await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
      }
    });
+    await AwsCliCommands.awsListLogGroups(async (element) => {
+      if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(new Date(element.createdAt)))) {
+        CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
+        await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
+      }
+    });
  }
 }
--- a/src/model/cloud-runner/providers/k8s/index.ts
+++ b/src/model/cloud-runner/providers/k8s/index.ts
@@ -119,7 +119,6 @@ class Kubernetes implements ProviderInterface {
            this.podName,
            'main',
            this.namespace,
-            CloudRunnerLogger.log,
          );
          break;
        } catch (error: any) {
--- a/src/model/cloud-runner/providers/k8s/kubernetes-task-runner.ts
+++ b/src/model/cloud-runner/providers/k8s/kubernetes-task-runner.ts
@@ -4,7 +4,7 @@ import CloudRunnerLogger from '../../services/cloud-runner-logger';
 import * as core from '@actions/core';
 import { CloudRunnerStatics } from '../../cloud-runner-statics';
 import waitUntil from 'async-wait-until';
-import CloudRunner from '../../cloud-runner';
+import { FollowLogStreamService } from '../../services/follow-log-stream-service';

 class KubernetesTaskRunner {
  static async runTask(
@@ -14,20 +14,23 @@ class KubernetesTaskRunner {
    podName: string,
    containerName: string,
    namespace: string,
-    logCallback: any,
  ) {
    CloudRunnerLogger.log(`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace}`);
    const stream = new Writable();
    let output = '';
    let didStreamAnyLogs: boolean = false;
+    let shouldReadLogs = true;
+    let shouldCleanup = true;
    stream._write = (chunk, encoding, next) => {
      didStreamAnyLogs = true;
      let message = chunk.toString().trimRight(`\n`);
      message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
-      if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
-        output += message;
-      }
-      logCallback(message);
+      ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
+        message,
+        shouldReadLogs,
+        shouldCleanup,
+        output,
+      ));
      next();
    };
    const logOptions = {