Commit 419b5e22 authored by Maximilien Colange's avatar Maximilien Colange

Test runner compares stats against nominal values, and reports discrepancies.

parent bcc9efea
......@@ -9,13 +9,34 @@ chomp $title;
my $call = <IN>;
chomp $call;
my $nominal ;
my $nominal;
my %expected;
while (my $line = <IN>) {
chomp $line;
if ($line =~ s/Result of verification is //) {
$nominal = $line;
last;
}
# gather stats
if ($line =~ s/^discrete states://) {
my @splitline = split(' ',$line);
$expected{"ds"} = $splitline[0];
}
if ($line =~ s/^states seen://) {
my @splitline = split(' ',$line);
$expected{"ss"} = $splitline[0];
}
if ($line =~ s/^states explored://) {
my @splitline = split(' ',$line);
$expected{"se"} = $splitline[0];
}
if ($line =~ s/^incl tests://) {
my @splitline = split(' ',$line);
$expected{"incl"} = $splitline[0];
}
if ($line =~ s/^pos\. incl tests://) {
my @splitline = split(' ',$line);
$expected{"pincl"} = $splitline[0];
}
}
......@@ -40,23 +61,38 @@ while (my $line = shift(@results)) {
# report some stats to teamcity
if ($line =~ s/^discrete states://) {
my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='discrete states' value='$splitline[0]']\n";
my $exp = $expected{"ds"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of discrete states $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
}
if ($line =~ s/^states seen://) {
my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='states seen' value='$splitline[0]']\n";
my $exp = $expected{"ss"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of states seen $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
}
if ($line =~ s/^states explored://) {
my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='states explored' value='$splitline[0]']\n";
my $exp = $expected{"se"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of states explored $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
}
if ($line =~ s/^incl tests://) {
my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='inclusion tests' value='$splitline[0]']\n";
my $exp = $expected{"incl"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of inclusion tests $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
}
if ($line =~ s/^pos\. incl tests://) {
my @splitline = split(' ',$line);
print "##teamcity[buildStatisticValue key='successful inclusion tests' value='$splitline[0]']\n";
my $exp = $expected{"pincl"};
if ($splitline[0] != $exp) {
print "##teamcity[message text='number of successful inclusiontests $splitline[0] differs from nominal $exp' status='WARNING']\n";
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment