Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PC: Refactor the console log/screenshot assets #21104

Merged
merged 1 commit into from
Feb 5, 2025
+94 −43
Merged

PC: Refactor the console log/screenshot assets #21104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 34 additions & 33 deletions lib/publiccloud/azure.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use utils qw(script_output_retry);
use publiccloud::azure_client;
use publiccloud::ssh_interactive 'select_host_console';
use Data::Dumper;
use DateTime;

has resource_group => 'openqa-upload';
has container => 'sle-images';
Expand Down Expand Up @@ -505,48 +506,51 @@ sub terraform_apply {
$args{vars}->{sku} = $sku if ($sku);

my @instances = $self->SUPER::terraform_apply(%args);
$self->upload_boot_diagnostics('resource_group' => $self->get_resource_group_from_terraform_show());

my $instance_id = $self->get_terraform_output('.instance_id.value[0]');
asmorodskyi marked this conversation as resolved.
Show resolved Hide resolved
$instance_id =~ s/.*\/(.*)/$1/;
record_info('INSTANCE_ID', $instance_id);
my $resource_group = $self->get_terraform_output('.resource_group_name.value[0]');
record_info('RESOURCE_GROUP', $resource_group);

$self->upload_boot_diagnostics('instance_id' => $instance_id, 'resource_group' => $resource_group);
return @instances;
}

sub on_terraform_apply_timeout {
my ($self) = @_;

my $resgroup = $self->get_resource_group_from_terraform_show();
return if (!defined($resgroup));

eval { $self->upload_boot_diagnostics('resource_group' => $resgroup) }
or record_info('Bootlog upl error', 'Failed to upload bootlog');
assert_script_run("az group delete --yes --no-wait --name $resgroup") unless get_var('PUBLIC_CLOUD_NO_CLEANUP');
}
my $instance_id = $self->get_terraform_output('.instance_id.value[0]');
$instance_id =~ s/.*\/(.*)/$1/;
record_info('INSTANCE_ID', $instance_id);
my $resource_group = $self->get_terraform_output('.resource_group_name.value[0]');
record_info('RESOURCE_GROUP', $resource_group);

sub get_resource_group_from_terraform_show {
my $resgroup;
my $out = script_output('terraform show -json');
eval {
my $json = decode_azure_json($out);
for my $resource (@{$json->{values}->{root_module}->{resources}}) {
next unless ($resource->{type} eq 'azurerm_resource_group');
$resgroup = $resource->{values}->{name};
last;
}
};
if ($@ || !defined($resgroup)) {
record_info('ERROR', "Unable to get resource-group:\n$out", result => 'fail');
}
return $resgroup;
$self->upload_boot_diagnostics('instance_id' => $instance_id, 'resource_group' => $resource_group);
assert_script_run("az group delete --yes --no-wait --name $resource_group") unless get_var('PUBLIC_CLOUD_NO_CLEANUP');
}

sub upload_boot_diagnostics {
my ($self, %args) = @_;
return if !defined($args{instance_id});
return if !defined($args{resource_group});
my $instance_id = $args{instance_id};
my $resource_group = $args{resource_group};

my $bootlog_name = '/tmp/azure-bootlog.txt';
my $cmd_enable = 'az vm boot-diagnostics enable --ids $(az vm list -g ' . $args{resource_group} . ' --query \'[].id\' -o tsv)';
my $dt = DateTime->now;
my $time = $dt->hms;
my $bootlog_name = "/tmp/bootlog-$time.txt";

my $cmd_enable = "az vm boot-diagnostics enable --ids $instance_id";
my $out = script_output($cmd_enable, 60 * 5, proceed_on_failure => 1);
record_info('INFO', $cmd_enable . $/ . $out);
assert_script_run('az vm boot-diagnostics get-boot-log --ids $(az vm list -g ' . $args{resource_group} . ' --query \'[].id\' -o tsv) | jq -r "." > ' . $bootlog_name);
upload_logs($bootlog_name, failok => 1);

script_run("timeout 110 az vm boot-diagnostics get-boot-log --name $instance_id --resource-group $resource_group | jq -r '.' > $bootlog_name", timeout => 120);
if (script_output("du $bootlog_name | cut -f1") < 8) {
record_soft_failure("poo#155116 - The console log is empty.");
} else {
upload_logs($bootlog_name, failok => 1);
}
}

sub on_terraform_destroy_timeout {
Expand Down Expand Up @@ -634,22 +638,19 @@ sub cleanup {
my ($self, $args) = @_;

script_run('cd ' . get_var('PUBLIC_CLOUD_TERRAFORM_DIR', '~/terraform'));
#my $terraform_output = script_output('terraform output -json', proceed_on_failure => 1);
#my $terraform_output_json = decode_json($terraform_output);
#my $instance_id = $terraform_output_json->{instance_id}->{value}[0];
my $instance_id = $self->get_terraform_output('.instance_id.value[0]');
$instance_id =~ s/.*\/(.*)/$1/;
#my $resource_group = $terraform_output_json->{resource_group_name}->{value}[0];
record_info('INSTANCE_ID', $instance_id);
my $resource_group = $self->get_terraform_output('.resource_group_name.value[0]');
record_info('RESOURCE_GROUP', $resource_group);
script_run('cd');

select_host_console(force => 1);

$self->get_image_version() if (get_var('PUBLIC_CLOUD_BUILD'));

if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id) && defined($resource_group)) {
script_run("timeout 110 az vm boot-diagnostics get-boot-log --name $instance_id --resource-group $resource_group | jq -r '.' > bootlog.txt", timeout => 120);
upload_logs("bootlog.txt", failok => 1);
$self->upload_boot_diagnostics('instance_id' => $instance_id, 'resource_group' => $resource_group);
}
$self->SUPER::cleanup();
}
Expand Down
38 changes: 31 additions & 7 deletions lib/publiccloud/ec2.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ use testapi;
use publiccloud::utils "is_byos";
use publiccloud::aws_client;
use publiccloud::ssh_interactive 'select_host_console';
use DateTime;

has ssh_key_pair => undef;
use constant SSH_KEY_PEM => 'QA_SSH_KEY.pem';
Expand Down Expand Up @@ -193,7 +194,35 @@ sub upload_img {
sub terraform_apply {
my ($self, %args) = @_;
$args{confidential_compute} = get_var("PUBLIC_CLOUD_CONFIDENTIAL_VM", 0);
return $self->SUPER::terraform_apply(%args);
my @instances = $self->SUPER::terraform_apply(%args);
asmorodskyi marked this conversation as resolved.
Show resolved Hide resolved
my $instance_id = $self->get_terraform_output('.vm_name.value[]');

if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id)) {
$self->upload_boot_diagnostics(instance_id => $instance_id);
}
return @instances;
}

sub upload_boot_diagnostics {
my ($self, %args) = @_;
return if !defined($args{instance_id});
my $instance_id = $args{instance_id};

my $dt = DateTime->now;
my $time = $dt->hms;
assert_script_run("aws ec2 get-console-output --latest --color=on --no-paginate --output text --instance-id $instance_id &> console-$time.txt");
if (script_output("du console-$time.txt | cut -f1") < 8) {
record_soft_failure('poo#155116 - The console log is empty.');
} else {
upload_logs("console-$time.txt", failok => 1);
}

script_run("aws ec2 get-console-screenshot --instance-id $instance_id | jq -r '.ImageData' | base64 --decode > console-$time.jpg");
if (script_output("du console-$time.jpg | cut -f1") < 8) {
record_info('empty screenshot', 'The console screenshot is empty.');
} else {
upload_logs("console-$time.jpg", failok => 1);
}
}

sub img_proof {
Expand All @@ -214,16 +243,11 @@ sub cleanup {
select_host_console(force => 1);

script_run('cd ' . get_var('PUBLIC_CLOUD_TERRAFORM_DIR', '~/terraform'));
#my $instance_id = script_output('terraform output -json | jq -r ".vm_name.value[0]"', proceed_on_failure => 1);
my $instance_id = $self->get_terraform_output('.vm_name.value[]');
script_run('cd');

if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id)) {
script_run("aws ec2 get-console-output --latest --color=on --no-paginate --output text --instance-id $instance_id &> console.txt");
upload_logs("console.txt", failok => 1);

script_run("aws ec2 get-console-screenshot --instance-id $instance_id | jq -r '.ImageData' | base64 --decode > console.jpg");
upload_logs("console.jpg", failok => 1);
$self->upload_boot_diagnostics(instance_id => $instance_id);
}
$self->terraform_destroy() if ($self->terraform_applied);
$self->delete_keypair();
Expand Down
32 changes: 29 additions & 3 deletions lib/publiccloud/gce.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use Mojo::JSON 'decode_json';
use testapi;
use utils;
use publiccloud::ssh_interactive 'select_host_console';
use DateTime;

sub init {
my ($self, %params) = @_;
Expand Down Expand Up @@ -85,9 +86,35 @@ sub terraform_apply {
my ($self, %args) = @_;
$args{project} //= $self->provider_client->project_id;
$args{confidential_compute} = get_var("PUBLIC_CLOUD_CONFIDENTIAL_VM", 0);
return $self->SUPER::terraform_apply(%args);
my @instances = $self->SUPER::terraform_apply(%args);

my $region = $self->{provider_client}->{region};
my $project = $self->{provider_client}->{project_id};
my $instance_id = $self->get_terraform_output(".vm_name.value[0]");
# gce provides full serial log, so extended timeout
if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id)) {
if ($instance_id =~ /$self->{resource_name}/) {
$self->upload_boot_diagnostics(project => $project, region => $region, instance_id => $instance_id);
} else {
record_info("Warn", "instance_id " . ($instance_id) ? $instance_id : "empty", result => 'fail');
}
}

return @instances;
}

sub upload_boot_diagnostics {
my ($self, %args) = @_;
return if !defined($args{instance_id});
my $project = $args{project};
my $region = $args{region};
my $instance_id = $args{instance_id};

my $dt = DateTime->now;
my $time = $dt->hms;
assert_script_run("gcloud compute --project=$project instances get-serial-port-output $instance_id --zone=$region --port=1 > instance_serial-$time.txt", timeout => 180);
upload_logs("instance_serial-$time.txt", failok => 1);
}

# In GCE we need to account for project name, if given
sub get_image_id {
Expand Down Expand Up @@ -174,8 +201,7 @@ sub cleanup {
# gce provides full serial log, so extended timeout
if (!check_var('PUBLIC_CLOUD_SLES4SAP', 1) && defined($instance_id)) {
if ($instance_id =~ /$self->{resource_name}/) {
script_run("gcloud compute --project=$project instances get-serial-port-output $instance_id --zone=$region --port=1 > instance_serial.txt", timeout => 180);
upload_logs("instance_serial.txt", failok => 1);
$self->upload_boot_diagnostics(project => $project, region => $region, instance_id => $instance_id);
} else {
record_info("Warn", "instance_id " . ($instance_id) ? $instance_id : "empty", result => 'fail');
}
Expand Down