}
# evaluate test case
+ set tmp_imagedir 0
if [catch {
# set variables identifying test case
uplevel set casename [file tail $casefile]
}
uplevel set imagedir \"$imagedir\"
+ set tmp_imagedir 1
}
# execute test scripts
# add memory and timing info
set stats ""
if { ! [catch {uplevel meminfo h} memuse] } {
- set stats "MEMORY DELTA: [expr ($memuse - $membase) / 1024] KiB\n"
+ append stats "MEMORY DELTA: [expr ($memuse - $membase) / 1024] KiB\n"
}
uplevel dchrono _timer stop
set time [uplevel dchrono _timer show]
- if [regexp -nocase {CPU user time:[ \t]*([0-9.e-]+)} $time res cpu] {
- set stats "${stats}TOTAL CPU TIME: $cpu sec\n"
+ if { [regexp -nocase {CPU user time:[ \t]*([0-9.e-]+)} $time res cpu_usr] } {
+ append stats "TOTAL CPU TIME: $cpu_usr sec\n"
}
if { $dlog_exists && ! $echo } {
dlog add $stats
} else {
puts $stats
}
+
+ # unset global vars
+ uplevel unset casename groupname gridname dirname
+ if { $tmp_imagedir } { uplevel unset imagedir test_image }
}
# Internal procedure to check log of test execution and decide if it passed or failed
continue
}
set status [string trim $status]
- if { $comment != "" } { set status "$status ([string trim $comment])" }
+ if { $comment != "" } { append status " ([string trim $comment])" }
set rexp [regsub -all {\\b} $rexp {\\y}] ;# convert regexp from Perl to Tcl style
lappend badwords [list $status $rexp]
}
} else {
while { [gets $fdlog logline] >= 0 } {
if { $include_cout } {
- set testout "$testout$logline\n"
+ append testout "$logline\n"
}
if [regexp -nocase {TOTAL CPU TIME:\s*([\d.]+)\s*sec} $logline res cpu] {
set add_cpu " time=\"$cpu\""
# record test case with its output and status
# Mapping is: SKIPPED, BAD, and OK to OK, all other to failure
- set testcases "$testcases\n <testcase name=\"$casename\"$add_cpu status=\"$result\">\n"
- set testcases "$testcases\n <system-out>\n$testout </system-out>"
+ append testcases "\n <testcase name=\"$casename\"$add_cpu status=\"$result\">\n"
+ append testcases "\n <system-out>\n$testout </system-out>"
if { $result != "OK" } {
if { [regexp -nocase {^SKIP} $result] } {
incr nberr
- set testcases "$testcases\n <error name=\"$result\" message=\"$message\"/>"
+ append testcases "\n <error name=\"$result\" message=\"$message\"/>"
} elseif { [regexp -nocase {^BAD} $result] } {
incr nbskip
- set testcases "$testcases\n <skipped>$message</skipped>"
+ append testcases "\n <skipped>$message</skipped>"
} else {
incr nbfail
- set testcases "$testcases\n <failure name=\"$result\" message=\"$message\"/>"
+ append testcases "\n <failure name=\"$result\" message=\"$message\"/>"
}
}
- set testcases "$testcases\n </testcase>"
+ append testcases "\n </testcase>"
}
# write last test suite
return [file join $dir1 $casepath "diff-[file tail $dir2]-$imgfile"]
}
+# auxiliary procedure to produce string comparing two values
+proc _diff_show_ratio {value1 value2} {
+ return "$value1 / $value2 \[[format "%+5.2f%%" [expr 100 * ($value1 - $value2) / double($value2)]]\]"
+}
+
# Procedure to compare results of two runs of test cases
proc _test_diff {dir1 dir2 basename status verbose _logvar {_statvar ""}} {
upvar $_logvar log
if { [llength $in1] > 0 } { _log_and_puts log "Only in $path1: $in1" }
if { [llength $in2] > 0 } { _log_and_puts log "Only in $path2: $in2" }
}
+ set gcpu1 0
+ set gcpu2 0
+ set gmem1 0
+ set gmem2 0
foreach logfile $common {
# load two logs
set log1 [_read_file [file join $dir1 $basename $logfile]]
[regexp {TOTAL CPU TIME:\s*([\d.]+)} $log2 res1 cpu2] } {
set stat(cpu1) [expr $stat(cpu1) + $cpu1]
set stat(cpu2) [expr $stat(cpu2) + $cpu2]
+ set gcpu1 [expr $gcpu1 + $cpu1]
+ set gcpu2 [expr $gcpu2 + $cpu2]
# compare CPU times with 10% precision (but not less 0.5 sec)
if { [expr abs ($cpu1 - $cpu2) > 0.5 + 0.05 * abs ($cpu1 + $cpu2)] } {
- _log_and_puts log "CPU [split $basename /] $casename: $cpu1 / $cpu2"
+ _log_and_puts log "CPU [split $basename /] $casename: [_diff_show_ratio $cpu1 $cpu2]"
}
}
[regexp {MEMORY DELTA:\s*([\d.]+)} $log2 res1 mem2] } {
set stat(mem1) [expr $stat(mem1) + $mem1]
set stat(mem2) [expr $stat(mem2) + $mem2]
+ set gmem1 [expr $gmem1 + $mem1]
+ set gmem2 [expr $gmem2 + $mem2]
# compare memory usage with 10% precision (but not less 16 KiB)
if { [expr abs ($mem1 - $mem2) > 16 + 0.05 * abs ($mem1 + $mem2)] } {
- _log_and_puts log "MEMORY [split $basename /] $casename: $mem1 / $mem2"
+ _log_and_puts log "MEMORY [split $basename /] $casename: [_diff_show_ratio $mem1 $mem2]"
}
}
# check images
- set imglist1 [glob -directory $path1 -types f -tails -nocomplain $casename*.{png,gif}]
- set imglist2 [glob -directory $path2 -types f -tails -nocomplain $casename*.{png,gif}]
+ set imglist1 [glob -directory $path1 -types f -tails -nocomplain ${casename}.{png,gif} ${casename}-*.{png,gif} ${casename}_*.{png,gif}]
+ set imglist2 [glob -directory $path2 -types f -tails -nocomplain ${casename}.{png,gif} ${casename}-*.{png,gif} ${casename}_*.{png,gif}]
_list_diff $imglist1 $imglist2 imgin1 imgin2 imgcommon
if { "$verbose" > 1 } {
if { [llength $imgin1] > 0 } { _log_and_puts log "Only in $path1: $imgin1" }
}
}
}
+
+ # report CPU and memory difference in group if it is greater than 10%
+ if { [expr abs ($gcpu1 - $gcpu2) > 0.5 + 0.005 * abs ($gcpu1 + $gcpu2)] } {
+ _log_and_puts log "CPU [split $basename /]: [_diff_show_ratio $gcpu1 $gcpu2]"
+ }
+ if { [expr abs ($gmem1 - $gmem2) > 16 + 0.005 * abs ($gmem1 + $gmem2)] } {
+ _log_and_puts log "MEMORY [split $basename /]: [_diff_show_ratio $gmem1 $gmem2]"
+ }
}
if { "$_statvar" == "" } {
- _log_and_puts log "Total MEMORY difference: $stat(mem1) / $stat(mem2)"
- _log_and_puts log "Total CPU difference: $stat(cpu1) / $stat(cpu2)"
+ _log_and_puts log "Total MEMORY difference: [_diff_show_ratio $stat(mem1) $stat(mem2)]"
+ _log_and_puts log "Total CPU difference: [_diff_show_ratio $stat(cpu1) $stat(cpu2)]"
}
}
puts $fd "<pre>"
set logpath [file split [file normalize $file]]
foreach line $log {
- puts $fd $line
+ # put a line; highlight considerable (>5%) deviations of CPU and memory
+ if { [regexp "\[\\\[](\[0-9.e+-]+)%\[\]]" $line res value] &&
+ [expr abs($value)] > 5 } {
+ puts $fd "<table><tr><td bgcolor=\"[expr $value > 0 ? \"red\" : \"lightgreen\"]\">$line</td></tr></table>"
+ } else {
+ puts $fd $line
+ }
+ # add images
if { [regexp {IMAGE[ \t]+([^:]+):[ \t]+([A-Za-z0-9_.-]+)} $line res case img] } {
if { [catch {eval file join "" [lrange $case 0 end-1]} gridpath] } {
# note: special handler for the case if test grid directoried are compared directly
projects / occt.git / blobdiff