Resolve some issues with "cpp_locals" (GH-4265)

* Fix class attributes access where cpp_locals=False.
* Add "no-cpp-locals" tag to mark a test as not suitable for running with "cpp_locals=True".
* Add a list of "extra_directives" to runtests as additional test mode.

Resolves some issues in https://github.com/cython/cython/issues/4266

36 files changed, 65 insertions, 37 deletions

diff --git a/tests/compile/cpp_temp_assignment.pyx b/tests/compile/cpp_temp_assignment.pyx
index 296fedce9..58ae39a70 100644
--- a/tests/compile/cpp_temp_assignment.pyx
+++ b/tests/compile/cpp_temp_assignment.pyx
@@ -1,5 +1,8 @@
 # tag: cpp,cpp11
-# mode: compile
+# mode: compile 
+# tag: no-cpp-locals
+# TODO cpp_locals works fine with the standard library that comes with gcc11
+# but not with gcc8. Therefore disable the test for now
 
 cdef extern from *:
     """
diff --git a/tests/errors/w_numpy_arr_as_cppvec_ref.pyx b/tests/errors/w_numpy_arr_as_cppvec_ref.pyx
index b8dd1f536..d3a70dbed 100644
--- a/tests/errors/w_numpy_arr_as_cppvec_ref.pyx
+++ b/tests/errors/w_numpy_arr_as_cppvec_ref.pyx
@@ -1,5 +1,5 @@
 # mode: error
-# tag: cpp, werror, numpy
+# tag: cpp, werror, numpy, no-cpp-locals
 
 import numpy as np
 cimport numpy as np
diff --git a/tests/errors/w_uninitialized_cpp.pyx b/tests/errors/w_uninitialized_cpp.pyx
index ae35978a8..a89c58da5 100644
--- a/tests/errors/w_uninitialized_cpp.pyx
+++ b/tests/errors/w_uninitialized_cpp.pyx
@@ -1,6 +1,7 @@
 # cython: warn.maybe_uninitialized=True
 # mode: error
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
+# FIXME - no-cpp-locals should work
 
 from cython.operator import typeid
 
diff --git a/tests/run/cpp_bool.pyx b/tests/run/cpp_bool.pyx
index b6027cd9d..98e281a2e 100644
--- a/tests/run/cpp_bool.pyx
+++ b/tests/run/cpp_bool.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from libcpp cimport bool
 
diff --git a/tests/run/cpp_class_redef.pyx b/tests/run/cpp_class_redef.pyx
index 8f0ae3ad5..36cd8ea04 100644
--- a/tests/run/cpp_class_redef.pyx
+++ b/tests/run/cpp_class_redef.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, warnings
+# tag: cpp, warnings, no-cpp-locals
 
 # This gives a warning about the previous .pxd definition, but should not give an error.
 cdef cppclass Foo:
diff --git a/tests/run/cpp_classes.pyx b/tests/run/cpp_classes.pyx
index bdaf4b7de..930a292b5 100644
--- a/tests/run/cpp_classes.pyx
+++ b/tests/run/cpp_classes.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from libcpp.vector cimport vector
 
diff --git a/tests/run/cpp_classes_def.pyx b/tests/run/cpp_classes_def.pyx
index 3bc8a1c4d..074d85a13 100644
--- a/tests/run/cpp_classes_def.pyx
+++ b/tests/run/cpp_classes_def.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp11
+# tag: cpp, werror, cpp11, no-cpp-locals
 # cython: experimental_cpp_class_def=True
 
 cdef double pi
diff --git a/tests/run/cpp_const_method.pyx b/tests/run/cpp_const_method.pyx
index ef1cada31..d959dbe09 100644
--- a/tests/run/cpp_const_method.pyx
+++ b/tests/run/cpp_const_method.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 # cython: experimental_cpp_class_def=True
 
 from libcpp.vector cimport vector
@@ -82,6 +82,6 @@ def test_vector_members(py_a, py_b):
 cdef vector_members(vector[const Wrapper[int]*] a, const vector[wrapInt*] b):
     # TODO: Cython-level error.
     # b[0].set(100)
-    
+
     # TODO: const_iterator
     return [x.get() for x in a], b[0].get()
diff --git a/tests/run/cpp_enums.pyx b/tests/run/cpp_enums.pyx
index 2c91d5187..ac5ea9fe7 100644
--- a/tests/run/cpp_enums.pyx
+++ b/tests/run/cpp_enums.pyx
@@ -1,5 +1,5 @@
 # tag: cpp
-# mode: run
+# mode: run, no-cpp-locals
 
 cdef extern from *:
     """
diff --git a/tests/run/cpp_exceptions_utility_code.pyx b/tests/run/cpp_exceptions_utility_code.pyx
index 91291f904..74f87dfb1 100644
--- a/tests/run/cpp_exceptions_utility_code.pyx
+++ b/tests/run/cpp_exceptions_utility_code.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 # ticket: 3065
 
 # This is intentionally in a file on its own. The issue was that it failed to generate utility-code
diff --git a/tests/run/cpp_forwarding_ref.pyx b/tests/run/cpp_forwarding_ref.pyx
index bf3469bc1..b3b044353 100644
--- a/tests/run/cpp_forwarding_ref.pyx
+++ b/tests/run/cpp_forwarding_ref.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, cpp11
+# tag: cpp, cpp11, no-cpp-locals
 
 from libcpp.utility cimport move
 
diff --git a/tests/run/cpp_iterators.pyx b/tests/run/cpp_iterators.pyx
index 36782710f..716fcb021 100644
--- a/tests/run/cpp_iterators.pyx
+++ b/tests/run/cpp_iterators.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from libcpp.deque cimport deque
 from libcpp.vector cimport vector
diff --git a/tests/run/cpp_optional_temps.pyx b/tests/run/cpp_locals_directive.pyx
index ca0580db1..6c9c89ba5 100644
--- a/tests/run/cpp_optional_temps.pyx
+++ b/tests/run/cpp_locals_directive.pyx
@@ -1,5 +1,6 @@
 # mode: run
-# tag: cpp, cpp17
+# tag: cpp, cpp17, no-cpp-locals
+# no-cpp-locals because the test is already run with it explicitly set
 
 # cython: cpp_locals=True
 
@@ -48,7 +49,7 @@ cdef extern from *:
     C make_C(int) except +  # needs a temp to receive
 
 # this function just makes sure the output from the destructor can be captured by doctest
-cdef void print_C_destructor "print_C_destructor" () nogil:
+cdef void print_C_destructor "print_C_destructor" () with gil:
     print("~C()")
 
 def maybe_assign_infer(assign, value, do_print):
@@ -167,11 +168,20 @@ cdef class HoldsC:
     >>> inst = HoldsC(True, False)
     >>> inst.getCX()
     10
+    >>> access_from_function_with_different_directive(inst)
+    10
+    10
+    >>> inst.getCX()  # it was changed in access_from_function_with_different_directive
+    20
     >>> inst = HoldsC(False, False)
     >>> inst.getCX()
     Traceback (most recent call last):
         ...
     AttributeError: C++ attribute 'value' is not initialized
+    >>> access_from_function_with_different_directive(inst)
+    Traceback (most recent call last):
+        ...
+    AttributeError: C++ attribute 'value' is not initialized
     """
     cdef C value
     def __cinit__(self, initialize, print_destructor):
@@ -181,6 +191,16 @@ cdef class HoldsC:
     def getCX(self):
         return self.value.getX()
 
+cdef acceptC(C& c):
+    return c.getX()
+
+@cython.cpp_locals(False)
+def access_from_function_with_different_directive(HoldsC c):
+    # doctest is in HoldsC class
+    print(acceptC(c.value))  # this originally tried to pass a __Pyx_Optional<C> as a C instance
+    print(c.value.getX())
+    c.value = C(20, False) # make sure that we can change it too
+
 def dont_test_on_pypy(f):
     import sys
     if not hasattr(sys, "pypy_version_info"):
diff --git a/tests/run/cpp_optional_temps_unused.pyx b/tests/run/cpp_locals_directive_unused.pyx
index e420a766d..429e64beb 100644
--- a/tests/run/cpp_optional_temps_unused.pyx
+++ b/tests/run/cpp_locals_directive_unused.pyx
@@ -1,5 +1,6 @@
 # mode: run
-# tag: cpp, cpp17
+# tag: cpp, cpp17, no-cpp-locals
+# no cpp_locals because this test is already run with cpp_locals explicitly set
 
 # cython: cpp_locals=True
 
diff --git a/tests/run/cpp_move.pyx b/tests/run/cpp_move.pyx
index d93afe28d..ca7cb7794 100644
--- a/tests/run/cpp_move.pyx
+++ b/tests/run/cpp_move.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp11
+# tag: cpp, werror, cpp11, no-cpp-locals
 
 from libcpp cimport nullptr
 from libcpp.memory cimport shared_ptr, make_shared
diff --git a/tests/run/cpp_nested_classes.pyx b/tests/run/cpp_nested_classes.pyx
index 98ea514f1..b50f79936 100644
--- a/tests/run/cpp_nested_classes.pyx
+++ b/tests/run/cpp_nested_classes.pyx
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 cdef extern from "cpp_nested_classes_support.h":
     cdef cppclass A:
diff --git a/tests/run/cpp_nonstdint.pyx b/tests/run/cpp_nonstdint.pyx
index 62153d190..238107b79 100644
--- a/tests/run/cpp_nonstdint.pyx
+++ b/tests/run/cpp_nonstdint.pyx
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 cdef extern from "cpp_nonstdint.h":
     ctypedef int Int24
diff --git a/tests/run/cpp_operator_exc_handling.pyx b/tests/run/cpp_operator_exc_handling.pyx
index 381870c22..67b00e39e 100644
--- a/tests/run/cpp_operator_exc_handling.pyx
+++ b/tests/run/cpp_operator_exc_handling.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from cython.operator import (preincrement, predecrement,
                              postincrement, postdecrement)
diff --git a/tests/run/cpp_operators.pyx b/tests/run/cpp_operators.pyx
index 7900aa064..f3ae6452a 100644
--- a/tests/run/cpp_operators.pyx
+++ b/tests/run/cpp_operators.pyx
@@ -1,5 +1,7 @@
 # mode: run
 # tag: cpp, werror
+# tag: no-cpp-locals
+# FIXME - cpp_locals should work but doesn't
 
 from __future__ import division
 
diff --git a/tests/run/cpp_smart_ptr.pyx b/tests/run/cpp_smart_ptr.pyx
index ec2276036..d71151c5e 100644
--- a/tests/run/cpp_smart_ptr.pyx
+++ b/tests/run/cpp_smart_ptr.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp11
+# tag: cpp, werror, cpp11, no-cpp-locals
 
 from libcpp.memory cimport unique_ptr, shared_ptr, default_delete, dynamic_pointer_cast
 from libcpp cimport nullptr
diff --git a/tests/run/cpp_static_method_overload.pyx b/tests/run/cpp_static_method_overload.pyx
index 25ce5bbb4..59eec2f44 100644
--- a/tests/run/cpp_static_method_overload.pyx
+++ b/tests/run/cpp_static_method_overload.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 cdef extern from *:
     """
diff --git a/tests/run/cpp_stl_algo_execpolicies.pyx b/tests/run/cpp_stl_algo_execpolicies.pyx
index 058232b5d..989e42f6c 100644
--- a/tests/run/cpp_stl_algo_execpolicies.pyx
+++ b/tests/run/cpp_stl_algo_execpolicies.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp17, cppexecpolicies
+# tag: cpp, werror, cpp17, cppexecpolicies, no-cpp-locals
 
 from libcpp.algorithm cimport is_sorted, sort, stable_sort, nth_element, all_of, count, copy
 from libcpp.execution cimport seq
diff --git a/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx b/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx
index f842d430b..0d823e288 100644
--- a/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx
+++ b/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp11
+# tag: cpp, werror, cpp11, no-cpp-locals
 
 from __future__ import print_function
 
diff --git a/tests/run/cpp_stl_algo_partitioning_ops.pyx b/tests/run/cpp_stl_algo_partitioning_ops.pyx
index 3bdc3d2af..1de80d84b 100644
--- a/tests/run/cpp_stl_algo_partitioning_ops.pyx
+++ b/tests/run/cpp_stl_algo_partitioning_ops.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp11
+# tag: cpp, werror, cpp11, no-cpp-locals
 
 from __future__ import print_function
 
diff --git a/tests/run/cpp_stl_algo_sorting_ops.pyx b/tests/run/cpp_stl_algo_sorting_ops.pyx
index 299089c4c..ac8d1b586 100644
--- a/tests/run/cpp_stl_algo_sorting_ops.pyx
+++ b/tests/run/cpp_stl_algo_sorting_ops.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp11
+# tag: cpp, werror, cpp11, no-cpp-locals
 
 from __future__ import print_function
 
diff --git a/tests/run/cpp_stl_atomic.pyx b/tests/run/cpp_stl_atomic.pyx
index e504c70eb..ba187a455 100644
--- a/tests/run/cpp_stl_atomic.pyx
+++ b/tests/run/cpp_stl_atomic.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, cpp11, werror
+# tag: cpp, cpp11, werror, no-cpp-locals
 
 from cython.operator cimport preincrement as incr, dereference as deref
 from libc.stdint cimport *
diff --git a/tests/run/cpp_stl_cpp11.pyx b/tests/run/cpp_stl_cpp11.pyx
index f4fa4d360..be0b72e33 100644
--- a/tests/run/cpp_stl_cpp11.pyx
+++ b/tests/run/cpp_stl_cpp11.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp11
+# tag: cpp, werror, cpp11, no-cpp-locals
 
 import sys
 from libcpp.unordered_map cimport unordered_map
diff --git a/tests/run/cpp_stl_forward_list.pyx b/tests/run/cpp_stl_forward_list.pyx
index 8ca081c4e..817e4d710 100644
--- a/tests/run/cpp_stl_forward_list.pyx
+++ b/tests/run/cpp_stl_forward_list.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror, cpp11
+# tag: cpp, werror, cpp11, no-cpp-locals
 
 from cython.operator cimport dereference as deref
 from cython.operator cimport preincrement as incr
diff --git a/tests/run/cpp_stl_list.pyx b/tests/run/cpp_stl_list.pyx
index 59367e2f5..6a3c60ee5 100644
--- a/tests/run/cpp_stl_list.pyx
+++ b/tests/run/cpp_stl_list.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from cython.operator cimport dereference as deref
 from cython.operator cimport preincrement as incr
diff --git a/tests/run/cpp_stl_vector.pyx b/tests/run/cpp_stl_vector.pyx
index 5c943e423..c42cb96b8 100644
--- a/tests/run/cpp_stl_vector.pyx
+++ b/tests/run/cpp_stl_vector.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from cython.operator cimport dereference as d
 from cython.operator cimport preincrement as incr
diff --git a/tests/run/cpp_template_functions.pyx b/tests/run/cpp_template_functions.pyx
index dce882879..19fa78413 100644
--- a/tests/run/cpp_template_functions.pyx
+++ b/tests/run/cpp_template_functions.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, warnings
+# tag: cpp, warnings, no-cpp-locals
 
 cimport cython
 from libcpp.pair cimport pair
diff --git a/tests/run/cpp_template_ref_args.pyx b/tests/run/cpp_template_ref_args.pyx
index c98c077ef..754122454 100644
--- a/tests/run/cpp_template_ref_args.pyx
+++ b/tests/run/cpp_template_ref_args.pyx
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 from libcpp.vector cimport vector
 
diff --git a/tests/run/cpp_template_subclasses.pyx b/tests/run/cpp_template_subclasses.pyx
index 0ebcd4c6e..3e7f3b506 100644
--- a/tests/run/cpp_template_subclasses.pyx
+++ b/tests/run/cpp_template_subclasses.pyx
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from cython.operator import dereference as deref
 from libcpp.pair cimport pair
diff --git a/tests/run/fused_cpp.pyx b/tests/run/fused_cpp.pyx
index 9f3bb5104..71ad9173a 100644
--- a/tests/run/fused_cpp.pyx
+++ b/tests/run/fused_cpp.pyx
@@ -1,4 +1,5 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
+# FIXME (or at least investigate) - cpp_locals should probably work
 
 cimport cython
 from libcpp.vector cimport vector
diff --git a/tests/run/libcpp_all.pyx b/tests/run/libcpp_all.pyx
index a45de63f6..6633adb7a 100644
--- a/tests/run/libcpp_all.pyx
+++ b/tests/run/libcpp_all.pyx
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 import cython
 
diff --git a/tests/run/lvalue_refs.pyx b/tests/run/lvalue_refs.pyx
index 6bd0d88bc..c70744533 100644
--- a/tests/run/lvalue_refs.pyx
+++ b/tests/run/lvalue_refs.pyx
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 from libcpp.vector cimport vector