Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Provide testcases result summary #516

Merged
merged 4 commits into from
Mar 13, 2024
Merged

Provide testcases result summary #516

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
887d69d
Test case results per test target
sophia-guo Mar 12, 2024
f1a2922
Add testcase summary for all test targets
sophia-guo Mar 12, 2024
85b4575
Update varialbe name
sophia-guo Mar 13, 2024
4d3c858
Using hash table for test cases summary
sophia-guo Mar 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 66 additions & 16 deletions scripts/resultsSum.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,15 @@ sub resultReporter {
my $numOfDisabled = 0;
my $numOfTotal = 0;
my $runningDisabled = 0;

my @passed;
my @failed;
my @disabled;
my @capSkipped;
my $tapString = '';
my $fhIn;
my @testCasesResults;
my $testCasesAllTargetsSummary;

print "\n\n";

Expand All @@ -130,6 +133,7 @@ sub resultReporter {
}
my $startTime = 0;
my $endTime = 0;
my $testCasesPerTargetSummary = '';
while ( $result = <$fhIn> ) {
# remove extra carriage return
$result =~ s/\r//g;
Expand All @@ -144,20 +148,34 @@ sub resultReporter {
$endTime = $1;
$tapString .= " duration_ms: " . ($endTime - $startTime) . "\n ...\n";
last;
} elsif ($result =~ /(Test results: .*)(passed|skipped|failed|error)(: \d{1,}$)/) {
$testCasesPerTargetSummary = $result;
push (@testCasesResults, $result);
} elsif ($result eq ($testName . "_PASSED\n")) {
push (@passed, $testName . " " . $successRate);
$numOfPassed++;
$numOfTotal++;
$tapString .= "ok " . $numOfTotal . " - " . $testName . "\n";
my $summarySuffix = '';
$tapString .= "ok " . $numOfTotal . " - " . $testName . "\n";
$tapString .= " ---\n";
if ( $testCasesPerTargetSummary ) {
$summarySuffix = " - " . $testCasesPerTargetSummary . " ";
$tapString .= " output:\n |\n";
$tapString .= " " . $testCasesPerTargetSummary;
}

push (@passed, $testName . $summarySuffix . $successRate);
if ($diagnostic eq 'all') {
$tapString .= $output;
}
} elsif ($result eq ($testName . "_FAILED\n")) {
push (@failed, $testName . " " . $successRate);
$numOfFailed++;
$numOfTotal++;
$tapString .= "not ok " . $numOfTotal . " - " . $testName . "\n";
my $summarySuffix = '';
if ( $testCasesPerTargetSummary ) {
$summarySuffix = " - " . $testCasesPerTargetSummary . " ";
}
push (@failed, $testName . $summarySuffix . $successRate);
$tapString .= "not ok " . $numOfTotal . " - " . $testName . "\n";
$tapString .= " ---\n";
# sometime jck test output shows after _FAILED message and before the $testName\E Finish Time
my $isJckFailedTestFinish = 0;
Expand Down Expand Up @@ -201,7 +219,6 @@ sub resultReporter {
my @testsInfo = split(/\s+/, $lines[$i]);
my $testName = $testsInfo[0];
$testName =~ s/#.*//;
print "Testname removed is " . $testName. "\n";
$failureTests .= ' ' ."TEST: " . $testName . "\n";
} elsif ( $lines[$i] =~ /(Test results: .*)(skipped|failed|error)(: \d{1,}$)/) {
$testResult = $lines[$i];
Expand Down Expand Up @@ -288,16 +305,24 @@ sub resultReporter {

$numOfExecuted = $numOfTotal - $numOfSkipped - $numOfDisabled;

my $testStatus = "TOTAL: $numOfTotal EXECUTED: $numOfExecuted PASSED: $numOfPassed FAILED: $numOfFailed";
my $testTargetStatus = "TOTAL: $numOfTotal EXECUTED: $numOfExecuted PASSED: $numOfPassed FAILED: $numOfFailed";
# Hide numOfDisabled when running disabled tests list.
if ($runningDisabled == 0) {
$testStatus .= " DISABLED: $numOfDisabled";
$testTargetStatus .= " DISABLED: $numOfDisabled";
}
$testStatus .= " SKIPPED: $numOfSkipped";
$testTargetStatus .= " SKIPPED: $numOfSkipped";
if ($comment ne "") {
$testStatus = "($testStatus)";
$testTargetStatus = "($testTargetStatus)";
}
print "$testTargetStatus\n";

print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n";
if (@testCasesResults) {
$testCasesAllTargetsSummary = getTestcaseResults(\@testCasesResults);
print $testCasesAllTargetsSummary . "\n";
print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n";
}
print "$testStatus\n";

if ($numOfTotal > 0) {
# set tap file name if not given
if ($tapName eq '') {
Expand Down Expand Up @@ -342,18 +367,19 @@ sub resultReporter {
print $fhOut "# CUSTOM_TARGET: " . "${customTarget}" . "\n";
}

print $fhOut "# RESULTS_SUMMARY: TOTAL: $numOfTotal EXECUTED: $numOfExecuted PASSED: $numOfPassed FAILED: $numOfFailed DISABLED: $numOfDisabled SKIPPED: $numOfSkipped\n";

print $fhOut "# TEST TARGETS RESULTS SUMMARY: $testTargetStatus\n";
if ( $testCasesAllTargetsSummary ) {
print $fhOut "# $testCasesAllTargetsSummary " . "\n";
}

print $fhOut "1.." . $numOfTotal . "\n";
print $fhOut $tapString;
close $fhOut;
if ($numOfFailed == 0) {
print "ALL TESTS PASSED\n";
}
}

print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n";


if ($numOfFailed != 0) {
my $buildParam = "";
if ($buildList ne '') {
Expand Down Expand Up @@ -392,11 +418,35 @@ sub resultReporter {
print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n";
}
}
unlink($resultFile);

unlink($resultFile);
return ($numOfTotal, \@failed);
}

sub getTestcaseResults() {
my $testCaseResults = '';
my @resultsArray = @{$_[0]};
my %testCasesSummary = ( 'passed: ' => 0, 'failed: ' => 0, 'error: ' => 0, 'skipped: ' => 0 );

for my $result (@resultsArray) {
$result =~ s/Test results: //;
my @statusNumbers = split(";", $result);
for my $statusNumber (@statusNumbers) {
$statusNumber =~ s/,//;
for (keys %testCasesSummary ) {
if ( $statusNumber =~ /\Q$_\E/) {
$statusNumber =~ s/\Q$_\E//;
$testCasesSummary{$_} += $statusNumber;
last;
}
}
}
}

$testCaseResults = "TESTCASES RESULTS SUMMARY: passed: " . $testCasesSummary{'passed: '} . "; failed: " . $testCasesSummary{'failed: '} . "; error: " . $testCasesSummary{'error: '} . "; skipped: " . $testCasesSummary{'skipped: '};
return $testCaseResults;
}

sub getTargetsFromResults() {
my $results = $_[0];
my @targets = ();
Expand Down
Loading