Commit 419b5e22 authored by Maximilien Colange's avatar Maximilien Colange

Test runner compares stats against nominal values, and reports discrepancies.

parent bcc9efea
...@@ -9,13 +9,34 @@ chomp $title; ...@@ -9,13 +9,34 @@ chomp $title;
my $call = <IN>; my $call = <IN>;
chomp $call; chomp $call;
my $nominal ; my $nominal;
my %expected;
while (my $line = <IN>) { while (my $line = <IN>) {
chomp $line; chomp $line;
if ($line =~ s/Result of verification is //) { if ($line =~ s/Result of verification is //) {
$nominal = $line; $nominal = $line;
last; }
# gather stats
if ($line =~ s/^discrete states://) {
my @splitline = split(' ',$line);
$expected{"ds"} = $splitline[0];
}
if ($line =~ s/^states seen://) {
my @splitline = split(' ',$line);
$expected{"ss"} = $splitline[0];
}
if ($line =~ s/^states explored://) {
my @splitline = split(' ',$line);
$expected{"se"} = $splitline[0];
}
if ($line =~ s/^incl tests://) {
my @splitline = split(' ',$line);
$expected{"incl"} = $splitline[0];
}
if ($line =~ s/^pos\. incl tests://) {
my @splitline = split(' ',$line);
$expected{"pincl"} = $splitline[0];
} }
} }
...@@ -40,23 +61,38 @@ while (my $line = shift(@results)) { ...@@ -40,23 +61,38 @@ while (my $line = shift(@results)) {
# report some stats to teamcity # report some stats to teamcity
if ($line =~ s/^discrete states://) { if ($line =~ s/^discrete states://) {
my @splitline = split(' ',$line); my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='discrete states' value='$splitline[0]']\n"; my $exp = $expected{"ds"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of discrete states $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
} }
if ($line =~ s/^states seen://) { if ($line =~ s/^states seen://) {
my @splitline = split(' ',$line); my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='states seen' value='$splitline[0]']\n"; my $exp = $expected{"ss"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of states seen $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
} }
if ($line =~ s/^states explored://) { if ($line =~ s/^states explored://) {
my @splitline = split(' ',$line); my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='states explored' value='$splitline[0]']\n"; my $exp = $expected{"se"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of states explored $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
} }
if ($line =~ s/^incl tests://) { if ($line =~ s/^incl tests://) {
my @splitline = split(' ',$line); my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='inclusion tests' value='$splitline[0]']\n"; my $exp = $expected{"incl"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of inclusion tests $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
} }
if ($line =~ s/^pos\. incl tests://) { if ($line =~ s/^pos\. incl tests://) {
my @splitline = split(' ',$line); my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='successful inclusion tests' value='$splitline[0]']\n"; my $exp = $expected{"pincl"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of successful inclusiontests $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment