Merge pull request #197 from rreilink/master

Fixes issues #195, 196

3 files changed, 73 insertions, 4 deletions

diff --git a/ply/cpp.py b/ply/cpp.py
index de3956b..50a44a1 100644
--- a/ply/cpp.py
+++ b/ply/cpp.py
@@ -617,11 +617,21 @@ class Preprocessor(object):
                 del tokens[i+1:j+1]
             i += 1
         tokens = self.expand_macros(tokens)
+        return self.evalexpr_expanded(tokens)
+
+    # ----------------------------------------------------------------------
+    # evalexpr_expanded()
+    #
+    # Helper for evalexpr that evaluates the expression that had its macros
+    # and defined(...) expressions expanded by evalexpr
+    # ----------------------------------------------------------------------
+
+    def evalexpr_expanded(self, tokens):
         for i,t in enumerate(tokens):
             if t.type == self.t_ID:
                 tokens[i] = copy.copy(t)
                 tokens[i].type = self.t_INTEGER
-                tokens[i].value = self.t_INTEGER_TYPE("0L")
+                tokens[i].value = self.t_INTEGER_TYPE("0")
             elif t.type == self.t_INTEGER:
                 tokens[i] = copy.copy(t)
                 # Strip off any trailing suffixes
@@ -629,10 +639,19 @@ class Preprocessor(object):
                 while tokens[i].value[-1] not in "0123456789abcdefABCDEF":
                     tokens[i].value = tokens[i].value[:-1]
 
-        expr = "".join([str(x.value) for x in tokens])
+        return self.evalexpr_string("".join([str(x.value) for x in tokens]))
+
+    # ----------------------------------------------------------------------
+    # evalexpr_string()
+    #
+    # Helper for evalexpr that evaluates a string expression
+    # This implementation does basic C->python conversion and then uses eval()
+    # ----------------------------------------------------------------------
+    def evalexpr_string(self, expr):
         expr = expr.replace("&&"," and ")
         expr = expr.replace("||"," or ")
         expr = expr.replace("!"," not ")
+        expr = expr.replace(" not ="," !=")
         try:
             result = eval(expr)
         except Exception:
@@ -805,8 +824,7 @@ class Preprocessor(object):
         for p in path:
             iname = os.path.join(p,filename)
             try:
-                with open(iname) as f:
-                    data = f.read()
+                data = self.read_include_file(iname)
                 dname = os.path.dirname(iname)
                 if dname:
                     self.temp_path.insert(0,dname)
@@ -821,6 +839,19 @@ class Preprocessor(object):
             print("Couldn't find '%s'" % filename)
 
     # ----------------------------------------------------------------------
+    # read_include_file()
+    #
+    # Reads a source file for inclusion using #include
+    # Could be overridden to e.g. customize encoding, limit access to
+    # certain paths on the filesystem, or provide the contents of system
+    # include files
+    # ----------------------------------------------------------------------
+
+    def read_include_file(self, filepath):
+        with open(filepath, 'r', encoding='utf-8', errors='surrogateescape') as file:
+            return file.read()
+
+    # ----------------------------------------------------------------------
     # define()
     #
     # Define a new macro
diff --git a/test/test_cpp_nonascii.c b/test/test_cpp_nonascii.c
new file mode 100644
index 0000000..3e97d81
--- /dev/null
+++ b/test/test_cpp_nonascii.c
@@ -0,0 +1,2 @@
+/* ë */
+#define x 1
\ No newline at end of file
diff --git a/test/testcpp.py b/test/testcpp.py
index 45478ff..dbfb3e4 100644
--- a/test/testcpp.py
+++ b/test/testcpp.py
@@ -4,6 +4,7 @@ from multiprocessing import Process, Queue
 from six.moves.queue import Empty
 
 import sys
+import locale
 
 if ".." not in sys.path:
     sys.path.insert(0, "..")
@@ -114,4 +115,39 @@ a"""
 a"""
         )
 
+    def test_evalexpr(self):
+        # #if 1 != 2 is not processed correctly; undefined values are converted
+        # to 0L instead of 0 (issue #195)
+        #
+        self.__test_preprocessing("""\
+#if (1!=0) && (!x || (!(1==2)))
+a;
+#else
+b;
+#endif
+"""
+            , """\
+
+a;
+
+"""
+        )
+
+    def test_include_nonascii(self):
+        # Issue #196: #included files are read using the current locale's
+        # getdefaultencoding. if a #included file contains non-ascii characters,
+        # while default encoding is e.g. US_ASCII, this causes an error
+        locale.setlocale(locale.LC_ALL, 'C')
+        self.__test_preprocessing("""\
+#include "test_cpp_nonascii.c"
+x;
+
+"""
+            , """\
+
+ 
+1;
+"""
+        )
+
 main()