Cloud runner develop - better parameterization of s3 usage, improved async workflow and GC, github checks early integration (#479)

* custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * add 3 new premade steps, steam-deploy-client, steam-deploy-project, aws-s3-pull-build * fix * fix * fix * continue building async-workflow support * test checks * test checks * test checks * move github checks within build workflow * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * cleanup logging * disable lz4 compression by default * disable lz4 compression by default * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * disable lz4 compression by default * disable lz4 compression by default * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * workflow * workflow * workflow * workflow * workflow * workflow * workflow * workflow * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3
2023-01-20 17:40:57 +00:00
parent a45155c578
commit e334dc785a
39 changed files with 3083 additions and 399 deletions
--- a/src/model/cloud-runner/providers/aws/aws-job-stack.ts
+++ b/src/model/cloud-runner/providers/aws/aws-job-stack.ts
@@ -5,6 +5,9 @@ import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
 import CloudRunnerLogger from '../../services/cloud-runner-logger';
 import { AWSError } from './aws-error';
 import CloudRunner from '../../cloud-runner';
+import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
+import CloudRunnerOptions from '../../cloud-runner-options';
+import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';

 export class AWSJobStack {
  private baseStackName: string;
@@ -38,6 +41,13 @@ export class AWSJobStack {
      `ContainerMemory:
    Default: ${Number.parseInt(memory)}`,
    );
+    if (CloudRunnerOptions.watchCloudRunnerToEnd) {
+      taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
+        taskDefCloudFormation,
+        '# template resources logstream',
+        TaskDefinitionFormation.streamLogs,
+      );
+    }
    for (const secret of secrets) {
      secret.ParameterKey = `${buildGuid.replace(/[^\dA-Za-z]/g, '')}${secret.ParameterKey.replace(
        /[^\dA-Za-z]/g,
@@ -57,7 +67,7 @@ export class AWSJobStack {
      );
      taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
        taskDefCloudFormation,
-        'p2 - secret',
+        '# template resources secrets',
        AWSCloudFormationTemplates.getSecretTemplate(`${secret.ParameterKey}`),
      );
      taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
@@ -132,14 +142,53 @@ export class AWSJobStack {
    };

    try {
+      CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
      await CF.createStack(createStackInput).promise();
-      CloudRunnerLogger.log('Creating cloud runner job');
      await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
    } catch (error) {
      await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
      throw error;
    }

+    const createCleanupStackInput: SDK.CloudFormation.CreateStackInput = {
+      StackName: `${taskDefStackName}-cleanup`,
+      TemplateBody: CleanupCronFormation.formation,
+      Capabilities: ['CAPABILITY_IAM'],
+      Parameters: [
+        {
+          ParameterKey: 'StackName',
+          ParameterValue: taskDefStackName,
+        },
+        {
+          ParameterKey: 'DeleteStackName',
+          ParameterValue: `${taskDefStackName}-cleanup`,
+        },
+        {
+          ParameterKey: 'TTL',
+          ParameterValue: `1080`,
+        },
+        {
+          ParameterKey: 'BUILDGUID',
+          ParameterValue: CloudRunner.buildParameters.buildGuid,
+        },
+        {
+          ParameterKey: 'EnvironmentName',
+          ParameterValue: this.baseStackName,
+        },
+      ],
+    };
+    if (CloudRunnerOptions.useCleanupCron) {
+      try {
+        CloudRunnerLogger.log(`Creating job cleanup formation`);
+        CF.createStack(createCleanupStackInput).promise();
+
+        // await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
+      } catch (error) {
+        await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
+        throw error;
+      }
+    }
+
    const taskDefResources = (
      await CF.describeStackResources({
        StackName: taskDefStackName,
--- a/src/model/cloud-runner/providers/aws/aws-task-runner.ts
+++ b/src/model/cloud-runner/providers/aws/aws-task-runner.ts
@@ -9,10 +9,12 @@ import CloudRunner from '../../cloud-runner';
 import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
 import { FollowLogStreamService } from '../../services/follow-log-stream-service';
 import CloudRunnerOptions from '../../cloud-runner-options';
+import GitHub from '../../../github';

 class AWSTaskRunner {
  public static ECS: AWS.ECS;
  public static Kinesis: AWS.Kinesis;
+  private static readonly encodedUnderscore = `$252F`;
  static async runTask(
    taskDef: CloudRunnerAWSTaskDef,
    environment: CloudRunnerEnvironmentVariable[],
@@ -56,7 +58,9 @@ class AWSTaskRunner {
    CloudRunnerLogger.log('Cloud runner job is starting');
    await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
    CloudRunnerLogger.log(
-      `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus}`,
+      `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Watch:${
+        CloudRunnerOptions.watchCloudRunnerToEnd
+      } Async:${CloudRunnerOptions.asyncCloudRunner}`,
    );
    if (!CloudRunnerOptions.watchCloudRunnerToEnd) {
      const shouldCleanup: boolean = false;
@@ -125,8 +129,9 @@ class AWSTaskRunner {
    const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
    let iterator = await AWSTaskRunner.getLogIterator(stream);

-    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
+    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
    CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
+    await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch:  ${logBaseUrl}`, ``);
    let shouldReadLogs = true;
    let shouldCleanup = true;
    let timestamp: number = 0;
--- a/src/model/cloud-runner/providers/aws/cloud-formations/base-stack-formation.ts
+++ b/src/model/cloud-runner/providers/aws/cloud-formations/base-stack-formation.ts
@@ -47,6 +47,11 @@ Resources:
      EnableDnsHostnames: true
      CidrBlock: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']

+  MainBucket:
+    Type: "AWS::S3::Bucket"
+    Properties:
+      BucketName: !Ref EnvironmentName
+
  EFSServerSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
--- a/src/model/cloud-runner/providers/aws/cloud-formations/cleanup-cron-formation.ts
+++ b/src/model/cloud-runner/providers/aws/cloud-formations/cleanup-cron-formation.ts
@@ -0,0 +1,146 @@
+export class CleanupCronFormation {
+  public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09'
+Description: Schedule automatic deletion of CloudFormation stacks
+Metadata:
+  AWS::CloudFormation::Interface:
+    ParameterGroups:
+      - Label:
+          default: Input configuration
+        Parameters:
+          - StackName
+          - TTL
+    ParameterLabels:
+      StackName:
+        default: Stack name
+      TTL:
+        default: Time-to-live
+Parameters:
+  EnvironmentName:
+    Type: String
+    Default: development
+    Description: 'Your deployment environment: DEV, QA , PROD'
+  BUILDGUID:
+    Type: String
+    Default: ''
+  StackName:
+    Type: String
+    Description: Stack name that will be deleted.
+  DeleteStackName:
+    Type: String
+    Description: Stack name that will be deleted.
+  TTL:
+    Type: Number
+    Description: Time-to-live in minutes for the stack.
+Resources:
+  DeleteCFNLambda:
+    Type: "AWS::Lambda::Function"
+    Properties:
+      FunctionName: !Join [ "", [ 'DeleteCFNLambda', !Ref BUILDGUID ] ]
+      Code:
+        ZipFile: |
+          import boto3
+          import os
+          import json
+
+          stack_name = os.environ['stackName']
+          delete_stack_name = os.environ['deleteStackName']
+
+          def delete_cfn(stack_name):
+              try:
+                  cfn = boto3.resource('cloudformation')
+                  stack = cfn.Stack(stack_name)
+                  stack.delete()
+                  return "SUCCESS"
+              except:
+                  return "ERROR"
+
+          def handler(event, context):
+              print("Received event:")
+              print(json.dumps(event))
+              result = delete_cfn(stack_name)
+              delete_cfn(delete_stack_name)
+              return result
+      Environment:
+        Variables:
+          stackName: !Ref 'StackName'
+          deleteStackName: !Ref 'DeleteStackName'
+      Handler: "index.handler"
+      Runtime: "python3.9"
+      Timeout: "5"
+      Role:
+       'Fn::ImportValue': !Sub '\${EnvironmentName}:DeleteCFNLambdaExecutionRole'
+  DeleteStackEventRule:
+     DependsOn:
+       - DeleteCFNLambda
+       - GenerateCronExpression
+     Type: "AWS::Events::Rule"
+     Properties:
+       Name: !Join [ "", [ 'DeleteStackEventRule', !Ref BUILDGUID ] ]
+       Description: Delete stack event
+       ScheduleExpression: !GetAtt GenerateCronExpression.cron_exp
+       State: "ENABLED"
+       Targets:
+          -
+            Arn: !GetAtt DeleteCFNLambda.Arn
+            Id: 'DeleteCFNLambda'
+  PermissionForDeleteCFNLambda:
+    Type: "AWS::Lambda::Permission"
+    DependsOn:
+      - DeleteStackEventRule
+    Properties:
+      FunctionName: !Join [ "", [ 'DeleteCFNLambda', !Ref BUILDGUID ] ]
+      Action: "lambda:InvokeFunction"
+      Principal: "events.amazonaws.com"
+      SourceArn: !GetAtt DeleteStackEventRule.Arn
+  GenerateCronExpLambda:
+    Type: "AWS::Lambda::Function"
+    Properties:
+      FunctionName: !Join [ "", [ 'GenerateCronExpressionLambda', !Ref BUILDGUID ] ]
+      Code:
+        ZipFile: |
+          from datetime import datetime, timedelta
+          import os
+          import logging
+          import json
+          import cfnresponse
+
+          def deletion_time(ttl):
+              delete_at_time = datetime.now() + timedelta(minutes=int(ttl))
+              hh = delete_at_time.hour
+              mm = delete_at_time.minute
+              yyyy = delete_at_time.year
+              month = delete_at_time.month
+              dd = delete_at_time.day
+              # minutes hours day month day-of-week year
+              cron_exp = "cron({} {} {} {} ? {})".format(mm, hh, dd, month, yyyy)
+              return cron_exp
+
+          def handler(event, context):
+            print('Received event: %s' % json.dumps(event))
+            status = cfnresponse.SUCCESS
+            try:
+                if event['RequestType'] == 'Delete':
+                    cfnresponse.send(event, context, status, {})
+                else:
+                    ttl = event['ResourceProperties']['ttl']
+                    responseData = {}
+                    responseData['cron_exp'] = deletion_time(ttl)
+                    cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
+            except Exception as e:
+                logging.error('Exception: %s' % e, exc_info=True)
+                status = cfnresponse.FAILED
+                cfnresponse.send(event, context, status, {}, None)
+      Handler: "index.handler"
+      Runtime: "python3.9"
+      Timeout: "5"
+      Role:
+       'Fn::ImportValue': !Sub '\${EnvironmentName}:DeleteCFNLambdaExecutionRole'
+  GenerateCronExpression:
+    Type: "Custom::GenerateCronExpression"
+    Version: "1.0"
+    Properties:
+      Name: !Join [ "", [ 'GenerateCronExpression', !Ref BUILDGUID ] ]
+      ServiceToken: !GetAtt GenerateCronExpLambda.Arn
+      ttl: !Ref 'TTL'
+`;
+}
--- a/src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts
+++ b/src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts
@@ -76,32 +76,10 @@ Resources:
    Metadata:
      'AWS::CloudFormation::Designer':
        id: aece53ae-b82d-4267-bc16-ed964b05db27
-  SubscriptionFilter:
-    Type: 'AWS::Logs::SubscriptionFilter'
-    Properties:
-      FilterPattern: ''
-      RoleArn:
-        'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:CloudWatchIAMRole'
-      LogGroupName: !Ref LogGroupName
-      DestinationArn:
-        'Fn::GetAtt':
-          - KinesisStream
-          - Arn
-    Metadata:
-      'AWS::CloudFormation::Designer':
-        id: 7f809e91-9e5d-4678-98c1-c5085956c480
-    DependsOn:
-      - LogGroup
-      - KinesisStream
-  KinesisStream:
-    Type: 'AWS::Kinesis::Stream'
-    Properties:
-      Name: !Ref ServiceName
-      ShardCount: 1
-    Metadata:
-      'AWS::CloudFormation::Designer':
-        id: c6f18447-b879-4696-8873-f981b2cedd2b
-  # template secrets p2 - secret
+  # template resources secrets
+
+  # template resources logstream
+
  TaskDefinition:
    Type: 'AWS::ECS::TaskDefinition'
    Properties:
@@ -156,5 +134,32 @@ Resources:
              awslogs-stream-prefix: !Ref ServiceName
    DependsOn:
      - LogGroup
+`;
+  public static streamLogs = `
+  SubscriptionFilter:
+    Type: 'AWS::Logs::SubscriptionFilter'
+    Properties:
+      FilterPattern: ''
+      RoleArn:
+        'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:CloudWatchIAMRole'
+      LogGroupName: !Ref LogGroupName
+      DestinationArn:
+        'Fn::GetAtt':
+          - KinesisStream
+          - Arn
+    Metadata:
+      'AWS::CloudFormation::Designer':
+        id: 7f809e91-9e5d-4678-98c1-c5085956c480
+    DependsOn:
+      - LogGroup
+      - KinesisStream
+  KinesisStream:
+    Type: 'AWS::Kinesis::Stream'
+    Properties:
+      Name: !Ref ServiceName
+      ShardCount: 1
+    Metadata:
+      'AWS::CloudFormation::Designer':
+        id: c6f18447-b879-4696-8873-f981b2cedd2b
 `;
 }
--- a/src/model/cloud-runner/providers/aws/index.ts
+++ b/src/model/cloud-runner/providers/aws/index.ts
@@ -13,6 +13,7 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 import { TaskService } from './services/task-service';
+import CloudRunnerOptions from '../../cloud-runner-options';

 class AWSBuildEnvironment implements ProviderInterface {
  private baseStackName: string;
@@ -133,6 +134,11 @@ class AWSBuildEnvironment implements ProviderInterface {
    await CF.deleteStack({
      StackName: taskDef.taskDefStackName,
    }).promise();
+    if (CloudRunnerOptions.useCleanupCron) {
+      await CF.deleteStack({
+        StackName: `${taskDef.taskDefStackName}-cleanup`,
+      }).promise();
+    }

    await CF.waitFor('stackDeleteComplete', {
      StackName: taskDef.taskDefStackName,
--- a/src/model/cloud-runner/providers/aws/services/task-service.ts
+++ b/src/model/cloud-runner/providers/aws/services/task-service.ts
@@ -4,6 +4,7 @@ import CloudRunnerLogger from '../../../services/cloud-runner-logger';
 import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
 import AwsTaskRunner from '../aws-task-runner';
 import { ListObjectsRequest } from 'aws-sdk/clients/s3';
+import CloudRunner from '../../../cloud-runner';

 export class TaskService {
  static async watch() {
@@ -158,7 +159,7 @@ export class TaskService {
    process.env.AWS_REGION = Input.region;
    const s3 = new AWS.S3();
    const listRequest: ListObjectsRequest = {
-      Bucket: `game-ci-test-storage`,
+      Bucket: CloudRunner.buildParameters.awsBaseStackName,
    };
    const results = await s3.listObjects(listRequest).promise();

--- a/src/model/cloud-runner/providers/docker/index.ts
+++ b/src/model/cloud-runner/providers/docker/index.ts
@@ -47,10 +47,18 @@ class LocalDockerCloudRunner implements ProviderInterface {
    defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
  ) {
    const { workspace } = Action;
-    if (fs.existsSync(`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar.lz4`)) {
+    if (
+      fs.existsSync(
+        `${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
+          CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        }`,
+      )
+    ) {
      await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
      await CloudRunnerSystem.Run(
-        `rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar.lz4`,
+        `rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
+          CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        }`,
      );
    }
  }
--- a/src/model/cloud-runner/providers/k8s/index.ts
+++ b/src/model/cloud-runner/providers/k8s/index.ts
@@ -154,6 +154,7 @@ class Kubernetes implements ProviderInterface {
          } else {
            CloudRunnerLogger.log('Pod still running, recovering stream...');
          }
+          await this.cleanupTaskResources();
        } catch (error: any) {
          let errorParsed;
          try {
@@ -161,10 +162,16 @@ class Kubernetes implements ProviderInterface {
          } catch {
            errorParsed = error;
          }
-          const reason = errorParsed.reason || errorParsed.response?.body?.reason || ``;
-          const errorMessage = errorParsed.message || ``;

-          const continueStreaming = reason === `NotFound` || errorMessage.includes(`dial timeout, backstop`);
+          const reason = errorParsed.reason || errorParsed.response?.body?.reason || ``;
+          const errorMessage = errorParsed.message || reason;
+
+          const continueStreaming =
+            errorMessage.includes(`dial timeout, backstop`) ||
+            errorMessage.includes(`HttpError: HTTP request failed`) ||
+            errorMessage.includes(`an error occurred when try to find container`) ||
+            errorMessage.includes(`not found`) ||
+            errorMessage.includes(`Not Found`);
          if (continueStreaming) {
            CloudRunnerLogger.log('Log Stream Container Not Found');
            await new Promise((resolve) => resolve(5000));
@@ -175,7 +182,6 @@ class Kubernetes implements ProviderInterface {
          }
        }
      }
-      await this.cleanupTaskResources();

      return output;
    } catch (error) {
--- a/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts
+++ b/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts
@@ -158,6 +158,8 @@ class KubernetesJobSpecFactory {
      },
    };

+    job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '5Gi';
+
    return job;
  }
 }
--- a/src/model/cloud-runner/providers/k8s/kubernetes-pods.ts
+++ b/src/model/cloud-runner/providers/k8s/kubernetes-pods.ts
@@ -4,8 +4,11 @@ class KubernetesPods {
  public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
    const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
    const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
-    const phase = pods[0].status?.phase || 'undefined status';
+    const phase = pods[0]?.status?.phase || 'undefined status';
    CloudRunnerLogger.log(`Getting pod status: ${phase}`);
+    if (phase === `Failed`) {
+      throw new Error(`K8s pod failed`);
+    }

    return running;
  }