0026589: Testgrid command - Add option to execute only tests failed on previous run

New option for command testgrid was added.
Updated documentation.

dox/dev_guides/tests/tests.md

@@ -109,6 +109,14 @@ Draw[]> testgrid -outdir d:/occt/last_results -overwrite
 ~~~~~
 In the output directory, a cumulative HTML report <i>summary.html</i> provides links to reports on each test case. An additional report in JUnit-style XML format can be output for use in Jenkins or other continuous integration system.
 
+To re-run test cases which were detected as regressions on previous run, option <i>-regress dirname</i> should be used.
+<i>dirname</i> is path to directory containing results of previous run. Only test cases with *FAILED* and *IMPROVEMENT* statuses will be tested.
+
+Example:
+~~~~~
+Draw[]> testgrid -regress d:/occt/last_results
+~~~~~
+
 Type <i>help testgrid</i> in DRAW prompt to get help on options supported by *testgrid* command:
 
 ~~~~~
@@ -122,6 +130,8 @@ testgrid: Run all tests, or specified group, or one grid
     -overwrite: force writing logs in existing non-empty directory
     -xml filename: write XML report for Jenkins (in JUnit-like format)
     -beep: play sound signal at the end of the tests
+    -regress dirname: re-run only a set of tests that have been detected as regressions on some previous run.
+                      Here "dirname" is path to directory containing results of previous run.
     Groups, grids, and test cases to be executed can be specified by list of file 
     masks, separated by spaces or comma; default is all (*).
 ~~~~~
@@ -630,11 +640,11 @@ puts "REQUIRED ListOfPlatforms: RegularExpression"
 
 Here *ListOfPlatforms* and *RegularExpression* have the same meaning as in TODO statements described above.
 
-The REQUIRED statament can also be used to mask message that would normally be interpreted as error (according to rules defined in *parse.rules*) but should not be considered as such within current test.
+The REQUIRED statement can also be used to mask message that would normally be interpreted as error (according to rules defined in *parse.rules*) but should not be considered as such within current test.
 
 Example:
 ~~~~~
-puts "TODO REQUIRED Linux: Faulty shapes in variables faulty_1 to faulty_5"
+puts "REQUIRED Linux: Faulty shapes in variables faulty_1 to faulty_5"
 ~~~~~
 
 This statement notifies test system that errors reported by *checkshape* command are expected in that test case, and test should be considered as OK if this message appears, despite of presence of general rule stating that 'Faulty' signals failure.

src/DrawResources/TestCommands.tcl

@@ -140,6 +140,8 @@ help testgrid {
   -overwrite: force writing logs in existing non-empty directory
   -xml filename: write XML report for Jenkins (in JUnit-like format)
   -beep: play sound signal at the end of the tests
+  -regress dirname: re-run only a set of tests that have been detected as regressions on some previous run.
+                    Here "dirname" is path to directory containing results of previous run.
   Groups, grids, and test cases to be executed can be specified by list of file 
   masks, separated by spaces or comma; default is all (*).
 }
@@ -163,6 +165,8 @@ proc testgrid {args} {
     set overwrite 0
     set xmlfile ""
     set signal 0
+    set regress 0
+    set prev_logdir ""
     for {set narg 0} {$narg < [llength $args]} {incr narg} {
         set arg [lindex $args $narg]
 
@@ -223,6 +227,18 @@ proc testgrid {args} {
             continue
         }
 
+        # re-run only a set of tests that have been detected as regressions on some previous run
+        if { $arg == "-regress" } {
+            incr narg
+            if { $narg < [llength $args] && ! [regexp {^-} [lindex $args $narg]] } {
+                set prev_logdir [lindex $args $narg]
+                set regress 1
+            } else {
+                error "Option -regress requires argument"
+            }
+            continue
+        }
+
         # unsupported option
         if { [regexp {^-} $arg] } {
             error "Error: unsupported option \"$arg\""
@@ -242,12 +258,13 @@ proc testgrid {args} {
 
     # check that target log directory is empty or does not exist
     set logdir [file normalize [string trim $logdir]]
+    set prev_logdir [file normalize [string trim $prev_logdir]]
     if { $logdir == "" } {
         # if specified logdir is empty string, generate unique name like 
         # results/<branch>_<timestamp>
         set prefix ""
         if { ! [catch {exec git branch} gitout] &&
-             [regexp {[*] ([\w]+)} $gitout res branch] } {
+             [regexp {[*] ([\w-]+)} $gitout res branch] } {
             set prefix "${branch}_"
         }
         set logdir "results/${prefix}[clock format [clock seconds] -format {%Y-%m-%dT%H%M}]"
@@ -270,6 +287,33 @@ proc testgrid {args} {
     if { ! [info exists gridmask ] } { set gridmask  * }
     if { ! [info exists casemask ] } { set casemask  * }
 
+    # Find test cases with FAILED and IMPROVEMENT statuses in previous run
+    # if option "regress" is given
+    set rerun_group_grid_case {}
+
+    if { ${regress} > 0 } {
+        if { "${groupmask}" != "*"} {
+            lappend rerun_group_grid_case [list $groupmask $gridmask $casemask]
+        }
+    } else {
+        lappend rerun_group_grid_case [list $groupmask $gridmask $casemask]
+    }
+
+    if { ${regress} > 0 } {
+        if { [file exists ${prev_logdir}/tests.log] } {
+            set fd [open ${prev_logdir}/tests.log]
+            while { [gets $fd line] >= 0 } {
+                if {[regexp {CASE ([^\s]+) ([^\s]+) ([^\s]+): FAILED} $line dump group grid casename] ||
+                    [regexp {CASE ([^\s]+) ([^\s]+) ([^\s]+): IMPROVEMENT} $line dump group grid casename]} {
+                    lappend rerun_group_grid_case [list $group $grid $casename]
+                }
+            }
+            close $fd
+        } else {
+            error "Error: file ${prev_logdir}/tests.log is not found, check your input arguments!"
+        }
+    }
+
     ######################################################
     # prepare list of tests to be performed
     ######################################################
@@ -282,99 +326,105 @@ proc testgrid {args} {
     # path to test case file
     set tests_list {}
 
-    # iterate by all script paths
+    foreach group_grid_case ${rerun_group_grid_case} {
-    foreach dir [lsort -unique [_split_path $env(CSF_TestScriptsPath)]] {
+        set groupmask [lindex $group_grid_case 0]
-        # protection against empty paths
+        set gridmask  [lindex $group_grid_case 1]
-        set dir [string trim $dir]
+        set casemask  [lindex $group_grid_case 2]
-        if { $dir == "" } { continue }
 
-        if { $_tests_verbose > 0 } { _log_and_puts log "Examining tests directory $dir" }
+        # iterate by all script paths
+        foreach dir [lsort -unique [_split_path $env(CSF_TestScriptsPath)]] {
+            # protection against empty paths
+            set dir [string trim $dir]
+            if { $dir == "" } { continue }
 
-        # check that directory exists
+            if { $_tests_verbose > 0 } { _log_and_puts log "Examining tests directory $dir" }
-        if { ! [file isdirectory $dir] } {
-            _log_and_puts log "Warning: directory $dir listed in CSF_TestScriptsPath does not exist, skipped"
-            continue
-        }
 
-        # search all directories in the current dir with specified mask
+            # check that directory exists
-        if [catch {glob -directory $dir -tail -types d {*}$groupmask} groups] { continue }
+            if { ! [file isdirectory $dir] } {
-
+                _log_and_puts log "Warning: directory $dir listed in CSF_TestScriptsPath does not exist, skipped"
-        # iterate by groups
-        if { $_tests_verbose > 0 } { _log_and_puts log "Groups to be executed: $groups" }
-        foreach group [lsort -dictionary $groups] {
-            if { $_tests_verbose > 0 } { _log_and_puts log "Examining group directory $group" }
-
-            # file grids.list must exist: it defines sequence of grids in the group
-            if { ! [file exists $dir/$group/grids.list] } {
-                _log_and_puts log "Warning: directory $dir/$group does not contain file grids.list, skipped"
                 continue
             }
 
-            # read grids.list file and make a list of grids to be executed
+            # search all directories in the current dir with specified mask
-            set gridlist {}
+            if [catch {glob -directory $dir -tail -types d {*}$groupmask} groups] { continue }
-            set fd [open $dir/$group/grids.list]
-            set nline 0
-            while { [gets $fd line] >= 0 } {
-                incr nline
 
-                # skip comments and empty lines
+            # iterate by groups
-                if { [regexp "\[ \t\]*\#.*" $line] } { continue }
+            if { $_tests_verbose > 0 } { _log_and_puts log "Groups to be executed: $groups" }
-                if { [string trim $line] == "" } { continue }
+            foreach group [lsort -dictionary $groups] {
+                if { $_tests_verbose > 0 } { _log_and_puts log "Examining group directory $group" }
 
-                # get grid id and name
+                # file grids.list must exist: it defines sequence of grids in the group
-                if { ! [regexp "^\(\[0-9\]+\)\[ \t\]*\(\[A-Za-z0-9_.-\]+\)\$" $line res gridid grid] } {
+                if { ! [file exists $dir/$group/grids.list] } {
-                    _log_and_puts log "Warning: cannot recognize line $nline in file $dir/$group/grids.list as \"gridid gridname\"; ignored"
+                    _log_and_puts log "Warning: directory $dir/$group does not contain file grids.list, skipped"
                     continue
                 }
 
-                # check that grid fits into the specified mask
+                # read grids.list file and make a list of grids to be executed
-                foreach mask $gridmask {
+                set gridlist {}
-                    if { $mask == $gridid || [string match $mask $grid] } {
+                set fd [open $dir/$group/grids.list]
-                        lappend gridlist $grid
+                set nline 0
-                    }
+                while { [gets $fd line] >= 0 } {
-                }
+                    incr nline
-            }
-            close $fd
 
-            # iterate by all grids
+                    # skip comments and empty lines
-            foreach grid $gridlist {
+                    if { [regexp "\[ \t\]*\#.*" $line] } { continue }
+                    if { [string trim $line] == "" } { continue }
 
-                # check if this grid is aliased to another one
+                    # get grid id and name
-                set griddir $dir/$group/$grid
+                    if { ! [regexp "^\(\[0-9\]+\)\[ \t\]*\(\[A-Za-z0-9_.-\]+\)\$" $line res gridid grid] } {
-                if { [file exists $griddir/cases.list] } {
+                        _log_and_puts log "Warning: cannot recognize line $nline in file $dir/$group/grids.list as \"gridid gridname\"; ignored"
-                    set fd [open $griddir/cases.list]
-                    if { [gets $fd line] >= 0 } {
-                        set griddir [file normalize $dir/$group/$grid/[string trim $line]]
-                    }
-                    close $fd
-                }
-
-                # check if grid directory actually exists
-                if { ! [file isdirectory $griddir] } { 
-                    _log_and_puts log "Error: tests directory for grid $grid ($griddir) is missing; skipped"
-                    continue 
-                }
-
-                # create directory for logging test results
-                if { $logdir != "" } { file mkdir $logdir/$group/$grid }
-
-                # iterate by all tests in the grid directory
-                if { [catch {glob -directory $griddir -type f {*}$casemask} testfiles] } { continue }
-                foreach casefile [lsort -dictionary $testfiles] {
-                    # filter out files with reserved names
-                    set casename [file tail $casefile]
-                    if { $casename == "begin" || $casename == "end" ||
-                         $casename == "parse.rules" } {
                         continue
                     }
 
-                    lappend tests_list [list $dir $group $grid $casename $casefile]
+                    # check that grid fits into the specified mask
+                    foreach mask $gridmask {
+                        if { $mask == $gridid || [string match $mask $grid] } {
+                            lappend gridlist $grid
+                        }
+                    }
+                }
+                close $fd
+
+                # iterate by all grids
+                foreach grid $gridlist {
+
+                    # check if this grid is aliased to another one
+                    set griddir $dir/$group/$grid
+                    if { [file exists $griddir/cases.list] } {
+                        set fd [open $griddir/cases.list]
+                        if { [gets $fd line] >= 0 } {
+                            set griddir [file normalize $dir/$group/$grid/[string trim $line]]
+                        }
+                        close $fd
+                    }
+
+                    # check if grid directory actually exists
+                    if { ! [file isdirectory $griddir] } {
+                        _log_and_puts log "Error: tests directory for grid $grid ($griddir) is missing; skipped"
+                        continue
+                    }
+
+                    # create directory for logging test results
+                    if { $logdir != "" } { file mkdir $logdir/$group/$grid }
+
+                    # iterate by all tests in the grid directory
+                    if { [catch {glob -directory $griddir -type f {*}$casemask} testfiles] } { continue }
+                    foreach casefile [lsort -dictionary $testfiles] {
+                        # filter out files with reserved names
+                        set casename [file tail $casefile]
+                        if { $casename == "begin" || $casename == "end" ||
+                             $casename == "parse.rules" } {
+                            continue
+                        }
+
+                        lappend tests_list [list $dir $group $grid $casename $casefile]
+                    }
                 }
             }
         }
     }
     if { [llength $tests_list] < 1 } {
-        error "Error: no tests are found, check you input arguments and variable CSF_TestScriptsPath!"
+        error "Error: no tests are found, check your input arguments and variable CSF_TestScriptsPath!"
     } else {
         puts "Running tests (total [llength $tests_list] test cases)..."
     }