diff --git a/lib/publiccloud/azure.pm b/lib/publiccloud/azure.pm index 6471d77c1152..b40a6a428a9d 100644 --- a/lib/publiccloud/azure.pm +++ b/lib/publiccloud/azure.pm @@ -17,6 +17,7 @@ use utils qw(script_output_retry); use publiccloud::azure_client; use publiccloud::ssh_interactive 'select_host_console'; use Data::Dumper; +use DateTime; has resource_group => 'openqa-upload'; has container => 'sle-images'; @@ -505,48 +506,51 @@ sub terraform_apply { $args{vars}->{sku} = $sku if ($sku); my @instances = $self->SUPER::terraform_apply(%args); - $self->upload_boot_diagnostics('resource_group' => $self->get_resource_group_from_terraform_show()); + + my $instance_id = $self->get_terraform_output('.instance_id.value[0]'); + $instance_id =~ s/.*\/(.*)/$1/; + record_info('INSTANCE_ID', $instance_id); + my $resource_group = $self->get_terraform_output('.resource_group_name.value[0]'); + record_info('RESOURCE_GROUP', $resource_group); + + $self->upload_boot_diagnostics('instance_id' => $instance_id, 'resource_group' => $resource_group); return @instances; } sub on_terraform_apply_timeout { my ($self) = @_; - my $resgroup = $self->get_resource_group_from_terraform_show(); - return if (!defined($resgroup)); - - eval { $self->upload_boot_diagnostics('resource_group' => $resgroup) } - or record_info('Bootlog upl error', 'Failed to upload bootlog'); - assert_script_run("az group delete --yes --no-wait --name $resgroup") unless get_var('PUBLIC_CLOUD_NO_CLEANUP'); -} + my $instance_id = $self->get_terraform_output('.instance_id.value[0]'); + $instance_id =~ s/.*\/(.*)/$1/; + record_info('INSTANCE_ID', $instance_id); + my $resource_group = $self->get_terraform_output('.resource_group_name.value[0]'); + record_info('RESOURCE_GROUP', $resource_group); -sub get_resource_group_from_terraform_show { - my $resgroup; - my $out = script_output('terraform show -json'); - eval { - my $json = decode_azure_json($out); - for my $resource (@{$json->{values}->{root_module}->{resources}}) { - next unless ($resource->{type} eq 'azurerm_resource_group'); - $resgroup = $resource->{values}->{name}; - last; - } - }; - if ($@ || !defined($resgroup)) { - record_info('ERROR', "Unable to get resource-group:\n$out", result => 'fail'); - } - return $resgroup; + $self->upload_boot_diagnostics('instance_id' => $instance_id, 'resource_group' => $resource_group); + assert_script_run("az group delete --yes --no-wait --name $resource_group") unless get_var('PUBLIC_CLOUD_NO_CLEANUP'); } sub upload_boot_diagnostics { my ($self, %args) = @_; + return if !defined($args{instance_id}); return if !defined($args{resource_group}); + my $instance_id = $args{instance_id}; + my $resource_group = $args{resource_group}; - my $bootlog_name = '/tmp/azure-bootlog.txt'; - my $cmd_enable = 'az vm boot-diagnostics enable --ids $(az vm list -g ' . $args{resource_group} . ' --query \'[].id\' -o tsv)'; + my $dt = DateTime->now; + my $time = $dt->hms; + my $bootlog_name = "/tmp/bootlog-$time.txt"; + + my $cmd_enable = "az vm boot-diagnostics enable --ids $instance_id"; my $out = script_output($cmd_enable, 60 * 5, proceed_on_failure => 1); record_info('INFO', $cmd_enable . $/ . $out); - assert_script_run('az vm boot-diagnostics get-boot-log --ids $(az vm list -g ' . $args{resource_group} . ' --query \'[].id\' -o tsv) | jq -r "." > ' . $bootlog_name); - upload_logs($bootlog_name, failok => 1); + + script_run("timeout 110 az vm boot-diagnostics get-boot-log --name $instance_id --resource-group $resource_group | jq -r '.' > $bootlog_name", timeout => 120); + if (script_output("du $bootlog_name | cut -f1") < 8) { + record_soft_failure("poo#155116 - The console log is empty."); + } else { + upload_logs($bootlog_name, failok => 1); + } } sub on_terraform_destroy_timeout { @@ -634,13 +638,11 @@ sub cleanup { my ($self, $args) = @_; script_run('cd ' . get_var('PUBLIC_CLOUD_TERRAFORM_DIR', '~/terraform')); - #my $terraform_output = script_output('terraform output -json', proceed_on_failure => 1); - #my $terraform_output_json = decode_json($terraform_output); - #my $instance_id = $terraform_output_json->{instance_id}->{value}[0]; my $instance_id = $self->get_terraform_output('.instance_id.value[0]'); $instance_id =~ s/.*\/(.*)/$1/; - #my $resource_group = $terraform_output_json->{resource_group_name}->{value}[0]; + record_info('INSTANCE_ID', $instance_id); my $resource_group = $self->get_terraform_output('.resource_group_name.value[0]'); + record_info('RESOURCE_GROUP', $resource_group); script_run('cd'); select_host_console(force => 1); @@ -648,8 +650,7 @@ sub cleanup { $self->get_image_version() if (get_var('PUBLIC_CLOUD_BUILD')); if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id) && defined($resource_group)) { - script_run("timeout 110 az vm boot-diagnostics get-boot-log --name $instance_id --resource-group $resource_group | jq -r '.' > bootlog.txt", timeout => 120); - upload_logs("bootlog.txt", failok => 1); + $self->upload_boot_diagnostics('instance_id' => $instance_id, 'resource_group' => $resource_group); } $self->SUPER::cleanup(); } diff --git a/lib/publiccloud/ec2.pm b/lib/publiccloud/ec2.pm index 673abdc178de..c1a4e439137e 100644 --- a/lib/publiccloud/ec2.pm +++ b/lib/publiccloud/ec2.pm @@ -14,6 +14,7 @@ use testapi; use publiccloud::utils "is_byos"; use publiccloud::aws_client; use publiccloud::ssh_interactive 'select_host_console'; +use DateTime; has ssh_key_pair => undef; use constant SSH_KEY_PEM => 'QA_SSH_KEY.pem'; @@ -193,7 +194,35 @@ sub upload_img { sub terraform_apply { my ($self, %args) = @_; $args{confidential_compute} = get_var("PUBLIC_CLOUD_CONFIDENTIAL_VM", 0); - return $self->SUPER::terraform_apply(%args); + my @instances = $self->SUPER::terraform_apply(%args); + my $instance_id = $self->get_terraform_output('.vm_name.value[]'); + + if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id)) { + $self->upload_boot_diagnostics(instance_id => $instance_id); + } + return @instances; +} + +sub upload_boot_diagnostics { + my ($self, %args) = @_; + return if !defined($args{instance_id}); + my $instance_id = $args{instance_id}; + + my $dt = DateTime->now; + my $time = $dt->hms; + assert_script_run("aws ec2 get-console-output --latest --color=on --no-paginate --output text --instance-id $instance_id &> console-$time.txt"); + if (script_output("du console-$time.txt | cut -f1") < 8) { + record_soft_failure('poo#155116 - The console log is empty.'); + } else { + upload_logs("console-$time.txt", failok => 1); + } + + script_run("aws ec2 get-console-screenshot --instance-id $instance_id | jq -r '.ImageData' | base64 --decode > console-$time.jpg"); + if (script_output("du console-$time.jpg | cut -f1") < 8) { + record_info('empty screenshot', 'The console screenshot is empty.'); + } else { + upload_logs("console-$time.jpg", failok => 1); + } } sub img_proof { @@ -214,16 +243,11 @@ sub cleanup { select_host_console(force => 1); script_run('cd ' . get_var('PUBLIC_CLOUD_TERRAFORM_DIR', '~/terraform')); - #my $instance_id = script_output('terraform output -json | jq -r ".vm_name.value[0]"', proceed_on_failure => 1); my $instance_id = $self->get_terraform_output('.vm_name.value[]'); script_run('cd'); if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id)) { - script_run("aws ec2 get-console-output --latest --color=on --no-paginate --output text --instance-id $instance_id &> console.txt"); - upload_logs("console.txt", failok => 1); - - script_run("aws ec2 get-console-screenshot --instance-id $instance_id | jq -r '.ImageData' | base64 --decode > console.jpg"); - upload_logs("console.jpg", failok => 1); + $self->upload_boot_diagnostics(instance_id => $instance_id); } $self->terraform_destroy() if ($self->terraform_applied); $self->delete_keypair(); diff --git a/lib/publiccloud/gce.pm b/lib/publiccloud/gce.pm index da00c5c53dc6..0f40a1d5a815 100644 --- a/lib/publiccloud/gce.pm +++ b/lib/publiccloud/gce.pm @@ -16,6 +16,7 @@ use Mojo::JSON 'decode_json'; use testapi; use utils; use publiccloud::ssh_interactive 'select_host_console'; +use DateTime; sub init { my ($self, %params) = @_; @@ -85,9 +86,35 @@ sub terraform_apply { my ($self, %args) = @_; $args{project} //= $self->provider_client->project_id; $args{confidential_compute} = get_var("PUBLIC_CLOUD_CONFIDENTIAL_VM", 0); - return $self->SUPER::terraform_apply(%args); + my @instances = $self->SUPER::terraform_apply(%args); + + my $region = $self->{provider_client}->{region}; + my $project = $self->{provider_client}->{project_id}; + my $instance_id = $self->get_terraform_output(".vm_name.value[0]"); + # gce provides full serial log, so extended timeout + if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id)) { + if ($instance_id =~ /$self->{resource_name}/) { + $self->upload_boot_diagnostics(project => $project, region => $region, instance_id => $instance_id); + } else { + record_info("Warn", "instance_id " . ($instance_id) ? $instance_id : "empty", result => 'fail'); + } + } + + return @instances; } +sub upload_boot_diagnostics { + my ($self, %args) = @_; + return if !defined($args{instance_id}); + my $project = $args{project}; + my $region = $args{region}; + my $instance_id = $args{instance_id}; + + my $dt = DateTime->now; + my $time = $dt->hms; + assert_script_run("gcloud compute --project=$project instances get-serial-port-output $instance_id --zone=$region --port=1 > instance_serial-$time.txt", timeout => 180); + upload_logs("instance_serial-$time.txt", failok => 1); +} # In GCE we need to account for project name, if given sub get_image_id { @@ -174,8 +201,7 @@ sub cleanup { # gce provides full serial log, so extended timeout if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id)) { if ($instance_id =~ /$self->{resource_name}/) { - script_run("gcloud compute --project=$project instances get-serial-port-output $instance_id --zone=$region --port=1 > instance_serial.txt", timeout => 180); - upload_logs("instance_serial.txt", failok => 1); + $self->upload_boot_diagnostics(project => $project, region => $region, instance_id => $instance_id); } else { record_info("Warn", "instance_id " . ($instance_id) ? $instance_id : "empty", result => 'fail'); }