0026589: Testgrid command - Add option to execute only tests failed on previous run

New option for command testgrid was added.
Updated documentation.

dox/dev_guides/tests/tests.md

@@ -109,6 +109,14 @@ Draw[]> testgrid -outdir d:/occt/last_results -overwrite
 ~~~~~
 In the output directory, a cumulative HTML report <i>summary.html</i> provides links to reports on each test case. An additional report in JUnit-style XML format can be output for use in Jenkins or other continuous integration system.
 
+To re-run test cases which were detected as regressions on previous run, option <i>-regress dirname</i> should be used.
+<i>dirname</i> is path to directory containing results of previous run. Only test cases with *FAILED* and *IMPROVEMENT* statuses will be tested.
+
+Example:
+~~~~~
+Draw[]> testgrid -regress d:/occt/last_results
+~~~~~
+
 Type <i>help testgrid</i> in DRAW prompt to get help on options supported by *testgrid* command:
 
 ~~~~~
@@ -122,6 +130,8 @@ testgrid: Run all tests, or specified group, or one grid
     -overwrite: force writing logs in existing non-empty directory
     -xml filename: write XML report for Jenkins (in JUnit-like format)
     -beep: play sound signal at the end of the tests
+    -regress dirname: re-run only a set of tests that have been detected as regressions on some previous run.
+                      Here "dirname" is path to directory containing results of previous run.
     Groups, grids, and test cases to be executed can be specified by list of file 
     masks, separated by spaces or comma; default is all (*).
 ~~~~~
@@ -630,11 +640,11 @@ puts "REQUIRED ListOfPlatforms: RegularExpression"
 
 Here *ListOfPlatforms* and *RegularExpression* have the same meaning as in TODO statements described above.
 
-The REQUIRED statament can also be used to mask message that would normally be interpreted as error (according to rules defined in *parse.rules*) but should not be considered as such within current test.
+The REQUIRED statement can also be used to mask message that would normally be interpreted as error (according to rules defined in *parse.rules*) but should not be considered as such within current test.
 
 Example:
 ~~~~~
-puts "TODO REQUIRED Linux: Faulty shapes in variables faulty_1 to faulty_5"
+puts "REQUIRED Linux: Faulty shapes in variables faulty_1 to faulty_5"
 ~~~~~
 
 This statement notifies test system that errors reported by *checkshape* command are expected in that test case, and test should be considered as OK if this message appears, despite of presence of general rule stating that 'Faulty' signals failure.

src/DrawResources/TestCommands.tcl

@@ -140,6 +140,8 @@ help testgrid {
   -overwrite: force writing logs in existing non-empty directory
   -xml filename: write XML report for Jenkins (in JUnit-like format)
   -beep: play sound signal at the end of the tests
+  -regress dirname: re-run only a set of tests that have been detected as regressions on some previous run.
+                    Here "dirname" is path to directory containing results of previous run.
   Groups, grids, and test cases to be executed can be specified by list of file 
   masks, separated by spaces or comma; default is all (*).
 }
@@ -163,6 +165,8 @@ proc testgrid {args} {
     set overwrite 0
     set xmlfile ""
     set signal 0
+    set regress 0
+    set prev_logdir ""
     for {set narg 0} {$narg < [llength $args]} {incr narg} {
         set arg [lindex $args $narg]
 
@@ -223,6 +227,18 @@ proc testgrid {args} {
             continue
         }
 
+        # re-run only a set of tests that have been detected as regressions on some previous run
+        if { $arg == "-regress" } {
+            incr narg
+            if { $narg < [llength $args] && ! [regexp {^-} [lindex $args $narg]] } {
+                set prev_logdir [lindex $args $narg]
+                set regress 1
+            } else {
+                error "Option -regress requires argument"
+            }
+            continue
+        }
+
         # unsupported option
         if { [regexp {^-} $arg] } {
             error "Error: unsupported option \"$arg\""
@@ -242,12 +258,13 @@ proc testgrid {args} {
 
     # check that target log directory is empty or does not exist
     set logdir [file normalize [string trim $logdir]]
+    set prev_logdir [file normalize [string trim $prev_logdir]]
     if { $logdir == "" } {
         # if specified logdir is empty string, generate unique name like 
         # results/<branch>_<timestamp>
         set prefix ""
         if { ! [catch {exec git branch} gitout] &&
-             [regexp {[*] ([\w]+)} $gitout res branch] } {
+             [regexp {[*] ([\w-]+)} $gitout res branch] } {
             set prefix "${branch}_"
         }
         set logdir "results/${prefix}[clock format [clock seconds] -format {%Y-%m-%dT%H%M}]"
@@ -270,6 +287,33 @@ proc testgrid {args} {
     if { ! [info exists gridmask ] } { set gridmask  * }
     if { ! [info exists casemask ] } { set casemask  * }
 
+    # Find test cases with FAILED and IMPROVEMENT statuses in previous run
+    # if option "regress" is given
+    set rerun_group_grid_case {}
+
+    if { ${regress} > 0 } {
+        if { "${groupmask}" != "*"} {
+            lappend rerun_group_grid_case [list $groupmask $gridmask $casemask]
+        }
+    } else {
+        lappend rerun_group_grid_case [list $groupmask $gridmask $casemask]
+    }
+
+    if { ${regress} > 0 } {
+        if { [file exists ${prev_logdir}/tests.log] } {
+            set fd [open ${prev_logdir}/tests.log]
+            while { [gets $fd line] >= 0 } {
+                if {[regexp {CASE ([^\s]+) ([^\s]+) ([^\s]+): FAILED} $line dump group grid casename] ||
+                    [regexp {CASE ([^\s]+) ([^\s]+) ([^\s]+): IMPROVEMENT} $line dump group grid casename]} {
+                    lappend rerun_group_grid_case [list $group $grid $casename]
+                }
+            }
+            close $fd
+        } else {
+            error "Error: file ${prev_logdir}/tests.log is not found, check your input arguments!"
+        }
+    }
+
     ######################################################
     # prepare list of tests to be performed
     ######################################################
@@ -282,6 +326,11 @@ proc testgrid {args} {
     # path to test case file
     set tests_list {}
 
+    foreach group_grid_case ${rerun_group_grid_case} {
+        set groupmask [lindex $group_grid_case 0]
+        set gridmask  [lindex $group_grid_case 1]
+        set casemask  [lindex $group_grid_case 2]
+
         # iterate by all script paths
         foreach dir [lsort -unique [_split_path $env(CSF_TestScriptsPath)]] {
             # protection against empty paths
@@ -373,8 +422,9 @@ proc testgrid {args} {
                 }
             }
         }
+    }
     if { [llength $tests_list] < 1 } {
-        error "Error: no tests are found, check you input arguments and variable CSF_TestScriptsPath!"
+        error "Error: no tests are found, check your input arguments and variable CSF_TestScriptsPath!"
     } else {
         puts "Running tests (total [llength $tests_list] test cases)..."
     }