From 3a0e01f6bb1d6ec444001f2caea6ef43a4a83e3a Mon Sep 17 00:00:00 2001 From: Jonathan Wakely Date: Fri, 1 Sep 2023 21:27:57 +0100 Subject: [PATCH] libstdc++: Add support for running tests with multiple -std options This copies the code from the compiler's gcc/testsuite/lib/g++-dg.exp so that each test can be run multiple times, with different -std options. This means that we can remove most { dg-options "-std=gnu++20" } directives from tests, because the testsuite will automatically select a set of -std options that includes that version. Tests that should only run for a specific standard (e.g. ones that use something like { dg-do run { target c++11_only } }) should still specify that standard with { dg-options "-std=gnu++11" }, which overrides the automatic selection. But a dg-options that selects a newer standard than the default can be removed, because that standard will be selected automatically based on a selector like { target c++20 } in the dg-do directive. This will allow those tests to be run for more than just the one they currently hardcode, so that e.g. std::format tests can be run for all of C++20, C++23 and C++26. Currently that has to be done by adding a second test file that uses a different dg-options line. By default most tests will continue to run with only the default dialect (currently -std=gnu++17) so that the time to run the entire testsuite is not increased. We can revisit this later if increasing the testsuite time (and coverage) is acceptable. Libstdc++ developers can easily override the defaults to run for multiple versions. To test all versions, either add 'set v3_std_list { 98 11 14 17 20 23 26 }' to ~/.dejagnurc or define GLIBCXX_TESTSUITE_STDS="98,11,14,17,20,23,26" in the environment. This should be more efficient than the current way to test with multple standards, i.e. --target_board=unix{-std=c++14,-std=c++17,-std=c++20}, because today all tests with an explicit -std option hardcoded in them get run for each target board variation but using the exact same hardcoded -std every time. With the new approach you can just use the default --target_board=unix and set GLIBCXX_TESTSUITE_STDS="14,17,20" and now a test that has { target c++20 } will only run once (and be UNSUPPORTED twice), instead of running with identical options three times. In order to support ~/.dejagnurc and $DEJAGNU files that need to work with versions of GCC without this change, a new variable is added to site.tmp to detect whether v3_std_list is supported. That allows e.g. if { [info exists v3-use-std-list] } { set v3_std_list { 11 17 23 } set target_list { "unix{,-m32}" } } else { set target_list { "unix{,-std=gnu++2b,-std=gnu++11,-m32}" } } libstdc++-v3/ChangeLog: * doc/xml/manual/test.xml: Update documentation on running and writing tests. * doc/html/manual/test.html: Regenerate. * testsuite/Makefile.am: Add v3-use-std-list to site.tmp * testsuite/Makefile.in: Regenerate. * testsuite/lib/dg-options.exp (add_options_for_strict_std): New proc. * testsuite/lib/libstdc++.exp (search_for): New utility proc. (v3-dg-runtest): New proc to replace dg-runtest. * testsuite/libstdc++-dg/conformance.exp: Use v3-dg-runtest. --- libstdc++-v3/doc/html/manual/test.html | 46 +++++--- libstdc++-v3/doc/xml/manual/test.xml | 52 ++++++--- libstdc++-v3/testsuite/Makefile.am | 1 + libstdc++-v3/testsuite/Makefile.in | 1 + libstdc++-v3/testsuite/lib/dg-options.exp | 7 ++ libstdc++-v3/testsuite/lib/libstdc++.exp | 107 ++++++++++++++++++ .../testsuite/libstdc++-dg/conformance.exp | 3 +- 7 files changed, 186 insertions(+), 31 deletions(-) diff --git a/libstdc++-v3/doc/html/manual/test.html b/libstdc++-v3/doc/html/manual/test.html index fadadf249bb..b29c2eb359e 100644 --- a/libstdc++-v3/doc/html/manual/test.html +++ b/libstdc++-v3/doc/html/manual/test.html @@ -350,6 +350,16 @@ cat 27_io/objects/char/3_xin.in | a.out
unix/-O3\"{-std=gnu++98,-std=gnu++11,}\" so that the third variation would use the default for -std (which is -std=gnu++14 as of GCC 6). +

+ Since GCC 14, the libstdc++ testsuite has built-in support for running + tests with more than one -std, similar to the G++ tests. + Adding set v3_std_list { 11 17 23 } to + ~/.dejagnurc or a file named by the + DEJAGNU environment variable will cause every test to + be run three times, using a different -std each time. + Alternatively, a list of standard versions to test with can be specified + as a comma-separated list in the <envvar>GLIBCXX_TESTSUITE_STDS</envvar> + environment variable.

To run the libstdc++ test suite under the debug mode, use @@ -467,11 +477,12 @@ cat 27_io/objects/char/3_xin.in | a.out

only be run for a specific standard (and not later standards) using an effective target like c++11_only. However, this means - the test will be skipped by default (because the default mode is - gnu++14), and so will only run when - -std=gnu++11 or -std=c++11 is used - explicitly. For tests that require a specific standard it is better to - use a dg-options directive: + the test will be skipped by default unless -std=gnu++11 + or -std=c++11 is explicitly specified, either via a + target board, the v3_std_list dejagnu variable, + or the <envvar>GLIBCXX_TESTSUITE_STDS</envvar> environment variable. + For tests that require a specific standard it is useful to also add a + dg-options directive: 

    // { dg-options "-std=gnu++11" }

This means the test will not get skipped by default, and will always use the specific standard dialect that the test requires. This isn't needed @@ -479,16 +490,21 @@ cat 27_io/objects/char/3_xin.in | a.out

- Similarly, tests which depend on a newer standard than the default - must use dg-options instead of (or in addition to) - an effective target, so that they are not skipped by default. - For example, tests for C++17 features should use -

    // { dg-options "-std=gnu++17" }

- before any dg-do such as: -

    // { dg-do run "c++17" }

- The dg-options directive must come first, so that - the -std flag has already been added to the options - before checking the c++17 target. + N.B. when a dg-options directive is used, it must come + first so dejagnu will include those options when checking against any + effective targets in dg-do and + dg-require-effective-target directives. +

+ Since GCC 14, tests which depend on a newer standard than the default + do not need to specify that standard in a dg-options + directive. The testsuite will detect when a test requires a newer standard + and will automatically add a suitable -std flag. +

+ If a testcase requires the use of a strict language dialect, e.g. + -std=c++11 rather than -std=gnu++11, + the following directive will cause that to be used when the testsuite + decides which -std options to use for the test: +

    // { dg-add-options strict_std }

Examples of Test Directives

Example 1: Testing compilation only: 

diff --git a/libstdc++-v3/doc/xml/manual/test.xml b/libstdc++-v3/doc/xml/manual/test.xml
index 964c53d2632..936f97417af 100644
--- a/libstdc++-v3/doc/xml/manual/test.xml
+++ b/libstdc++-v3/doc/xml/manual/test.xml
@@ -598,6 +598,18 @@ cat 27_io/objects/char/3_xin.in | a.out
       (which is  as of GCC 6).
     
 
+    
+      Since GCC 14, the libstdc++ testsuite has built-in support for running
+      tests with more than one , similar to the G++ tests.
+      Adding set v3_std_list { 11 17 23 } to
+      ~/.dejagnurc or a file named by the
+      DEJAGNU environment variable will cause every test to
+      be run three times, using a different  each time.
+      Alternatively, a list of standard versions to test with can be specified
+      as a comma-separated list in the GLIBCXX_TESTSUITE_STDS
+      environment variable.
+    
+
     
       To run the libstdc++ test suite under the
       debug mode, use
@@ -766,11 +778,12 @@ cat 27_io/objects/char/3_xin.in | a.out
     It is possible to indicate that a test should only
     be run for a specific standard (and not later standards) using an
     effective target like c++11_only. However, this means
-    the test will be skipped by default (because the default mode is
-    gnu++14), and so will only run when
-     or  is used
-    explicitly. For tests that require a specific standard it is better to
-    use a dg-options directive:
+    the test will be skipped by default unless 
+    or  is explicitly specified, either via a
+    target board, the v3_std_list dejagnu variable,
+    or the GLIBCXX_TESTSUITE_STDS environment variable.
+    For tests that require a specific standard it is useful to also add a
+    dg-options directive:
     // { dg-options "-std=gnu++11" }
     This means the test will not get skipped by default, and will always use
     the specific standard dialect that the test requires. This isn't needed
@@ -780,16 +793,25 @@ cat 27_io/objects/char/3_xin.in | a.out
   
 
   
-    Similarly, tests which depend on a newer standard than the default
-    must use dg-options instead of (or in addition to)
-    an effective target, so that they are not skipped by default.
-    For example, tests for C++17 features should use
-    // { dg-options "-std=gnu++17" }
-    before any dg-do such as:
-    // { dg-do run "c++17" }
-    The dg-options directive must come first, so that
-    the -std flag has already been added to the options
-    before checking the c++17 target.
+    N.B. when a dg-options directive is used, it must come
+    first so dejagnu will include those options when checking against any
+    effective targets in dg-do and
+    dg-require-effective-target directives.
+  
+
+  
+    Since GCC 14, tests which depend on a newer standard than the default
+    do not need to specify that standard in a dg-options
+    directive. The testsuite will detect when a test requires a newer standard
+    and will automatically add a suitable  flag.
+  
+
+  
+    If a testcase requires the use of a strict language dialect, e.g.
+     rather than ,
+    the following directive will cause that to be used when the testsuite
+    decides which  options to use for the test:
+    // { dg-add-options strict_std }
   
 
 
Examples of Test Directives
diff --git a/libstdc++-v3/testsuite/Makefile.am b/libstdc++-v3/testsuite/Makefile.am
index ef579d97efa..4cee585fd8e 100644
--- a/libstdc++-v3/testsuite/Makefile.am
+++ b/libstdc++-v3/testsuite/Makefile.am
@@ -61,6 +61,7 @@ site.exp: Makefile
 	@echo 'set baseline_dir "$(baseline_dir)"' >> site.tmp
 	@echo 'set baseline_subdir_switch "$(baseline_subdir_switch)"' >> site.tmp
 	@echo 'set TEST_GCC_EXEC_PREFIX "$(libdir)/gcc/"' >> site.tmp
+	@echo 'set v3-use-std-list 1' >> site.tmp
 	@echo '## All variables above are generated by configure. Do Not Edit ##' >>site.tmp
 	@test ! -f site.exp || \
 	  sed '1,/^## All variables above are.*##/ d' site.exp >> site.tmp
diff --git a/libstdc++-v3/testsuite/Makefile.in b/libstdc++-v3/testsuite/Makefile.in
index 462db4fd709..2e42d470645 100644
--- a/libstdc++-v3/testsuite/Makefile.in
+++ b/libstdc++-v3/testsuite/Makefile.in
@@ -624,6 +624,7 @@ site.exp: Makefile
 	@echo 'set baseline_dir "$(baseline_dir)"' >> site.tmp
 	@echo 'set baseline_subdir_switch "$(baseline_subdir_switch)"' >> site.tmp
 	@echo 'set TEST_GCC_EXEC_PREFIX "$(libdir)/gcc/"' >> site.tmp
+	@echo 'set v3-use-std-list 1' >> site.tmp
 	@echo '## All variables above are generated by configure. Do Not Edit ##' >>site.tmp
 	@test ! -f site.exp || \
 	  sed '1,/^## All variables above are.*##/ d' site.exp >> site.tmp
diff --git a/libstdc++-v3/testsuite/lib/dg-options.exp b/libstdc++-v3/testsuite/lib/dg-options.exp
index e4975f3e8ba..84ad0c65330 100644
--- a/libstdc++-v3/testsuite/lib/dg-options.exp
+++ b/libstdc++-v3/testsuite/lib/dg-options.exp
@@ -352,6 +352,13 @@ proc add_options_for_using-deprecated { flags } {
     return "$flags -U_GLIBCXX_USE_DEPRECATED -D_GLIBCXX_USE_DEPRECATED=1"
 }
 
+# Add options for strict -std=c++NN dialects.
+# This is a dummy option that does nothing, but it causes v3-dg-runtest
+# to alter its behaviour.
+proc add_options_for_strict_std { flags } {
+    return $flags
+}
+
 # Like dg-options, but adds to the default options rather than replacing them.
 
 proc dg-additional-options { args } {
diff --git a/libstdc++-v3/testsuite/lib/libstdc++.exp b/libstdc++-v3/testsuite/lib/libstdc++.exp
index fde687d8bc1..608056e5068 100644
--- a/libstdc++-v3/testsuite/lib/libstdc++.exp
+++ b/libstdc++-v3/testsuite/lib/libstdc++.exp
@@ -44,6 +44,21 @@ proc load_gcc_lib { filename } {
     set loaded_libs($filename) ""
 }
 
+#
+# search_for -- looks for a string match in a file
+#
+proc search_for { file pattern } {
+    set fd [open $file r]
+    while { [gets $fd cur_line]>=0 } {
+	if [string match "*$pattern*" $cur_line] then {
+	    close $fd
+	    return 1
+	}
+    }
+    close $fd
+    return 0
+}
+
 # system routines
 load_lib dg.exp
 load_lib libgloss.exp
@@ -460,6 +475,98 @@ if { [info procs saved-dg-test] == [list] } {
     }
 }
 
+# Find the minimum standard required by a test, if higher than the default_std.
+proc v3-minimum-std { test default_std max_std } {
+    for {set s $default_std} {$s <= $max_std} {incr s 3} {
+	if [search_for $test "\{ dg-do * \{ target c++$s"] {
+	    return $s
+	} elseif [search_for $test "\{ dg-require-effective-target c++$s"] {
+	    return $s
+	}
+    }
+    # TODO: replace all c++2a with c++20 and remove this case.
+    if [search_for $test "\{ dg-do * \{ target c++2a"] {
+	return 20
+    }
+    return $default_std
+}
+
+# Allow v3_std_list to be set in configuration files, e.g., ~/.dejagnurc
+if ![info exists v3_std_list] {
+    set v3_std_list { }
+}
+# Allow v3_std_list to be set from the environment.
+if [info exists env(GLIBCXX_TESTSUITE_STDS)] {
+    set v3_std_list [split $env(GLIBCXX_TESTSUITE_STDS) ","]
+}
+
+# Modified dg-runtest that runs tests in multiple standard modes,
+# unless they specifically specify one standard.
+proc v3-dg-runtest { testcases flags default-extra-flags } {
+    global runtests
+
+    foreach test $testcases {
+	# If we're only testing specific files and this isn't one of them, skip it.
+	if ![runtest_file_p $runtests $test] {
+	    continue
+	}
+
+	# If the testcase specifies a standard, use that one.
+	# If not, run it under several standards, allowing GNU extensions
+	# unless strict_std is requested.
+	if ![search_for $test "// \{ dg-*options*-std=*++"] {
+	    if [search_for $test "{ dg-add-options strict_std }"] {
+		set std_prefix "-std=c++"
+	    } else {
+		set std_prefix "-std=gnu++"
+	    }
+
+	    # See above for the initial value of this list.
+	    global v3_std_list
+	    if { [llength $v3_std_list] > 0 } {
+		set std_list $v3_std_list
+	    } else {
+		# If the test requires a newer C++ version than which
+		# is tested by default, use that C++ version for that
+		# single test.
+		# These should be adjusted whenever the default -std is
+		# updated or newer C++ effective target is added.
+		set default_std 17
+		set max_std 26
+		set min_std [v3-minimum-std $test $default_std $max_std]
+		if { $min_std > $default_std } {
+		    set std_list $min_std
+		    if { $min_std != $max_std } {
+			# Also test the latest version.
+			lappend std_list "$max_std"
+		    }
+		} else {
+		    # Only run each test once with the default -std option.
+		    # This avoids increasing the run time for most testers.
+		    # Libstdc++ developers can override this with v3_std_list.
+		    set std_list $default_std
+		}
+	    }
+	    set option_list { }
+	    foreach x $std_list {
+		if { $x eq "impcx" } then { set x "26 -fimplicit-constexpr" }
+		lappend option_list "${std_prefix}$x"
+	    }
+	} else {
+	    verbose "using -std option specified in [file tail $test]" 2
+	    set option_list { "" }
+	}
+
+	set nshort [file tail [file dirname $test]]/[file tail $test]
+
+	foreach flags_t $option_list {
+	    verbose "Testing $nshort, $flags $flags_t" 1
+	    dg-test $test "$flags $flags_t" ${default-extra-flags}
+	}
+    }
+}
+
+
 # True if the library supports wchar_t.
 set v3-wchar_t 0
 
diff --git a/libstdc++-v3/testsuite/libstdc++-dg/conformance.exp b/libstdc++-v3/testsuite/libstdc++-dg/conformance.exp
index 32d6f69fe89..76679739823 100644
--- a/libstdc++-v3/testsuite/libstdc++-dg/conformance.exp
+++ b/libstdc++-v3/testsuite/libstdc++-dg/conformance.exp
@@ -108,7 +108,8 @@ set tests [lsort $tests]
 # Main loop.
 global DEFAULT_CXXFLAGS
 global PCH_CXXFLAGS
-dg-runtest $tests "" "$DEFAULT_CXXFLAGS $PCH_CXXFLAGS"
+# Modified version of dg-runtest
+v3-dg-runtest $tests "" "$DEFAULT_CXXFLAGS $PCH_CXXFLAGS"
 
 # Finally run simd tests with extra SIMD-relevant flags
 global DEFAULT_VECTCFLAGS