Merge from 3.1: Issue #13703: add a way to randomize the hash values of basic types (str, bytes, datetime)

in order to make algorithmic complexity attacks on (e.g.) web apps much more complicated.

The environment variable PYTHONHASHSEED and the new command line flag -R control this
behavior.

1 files changed, 91 insertions, 36 deletions

diff --git a/Python/_warnings.c b/Python/_warnings.c
index 543e0bb4ca..c12db44d19 100644
--- a/Python/_warnings.c
+++ b/Python/_warnings.c
@@ -251,7 +251,7 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyObject
 
     name = PyObject_GetAttrString(category, "__name__");
     if (name == NULL)  /* XXX Can an object lack a '__name__' attribute? */
-            return;
+        return;
 
     f_stderr = PySys_GetObject("stderr");
     if (f_stderr == NULL) {
@@ -461,7 +461,7 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno,
     }
     else {
         globals = f->f_globals;
-        *lineno = PyCode_Addr2Line(f->f_code, f->f_lasti);
+        *lineno = PyFrame_GetLineNumber(f);
     }
 
     *module = NULL;
@@ -496,29 +496,28 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno,
 
     /* Setup filename. */
     *filename = PyDict_GetItemString(globals, "__file__");
-    if (*filename != NULL) {
+    if (*filename != NULL && PyUnicode_Check(*filename)) {
         Py_ssize_t len = PyUnicode_GetSize(*filename);
-        const char *file_str = _PyUnicode_AsString(*filename);
-            if (file_str == NULL || (len < 0 && PyErr_Occurred()))
-            goto handle_error;
+        Py_UNICODE *unicode = PyUnicode_AS_UNICODE(*filename);
 
         /* if filename.lower().endswith((".pyc", ".pyo")): */
         if (len >= 4 &&
-            file_str[len-4] == '.' &&
-            tolower(file_str[len-3]) == 'p' &&
-            tolower(file_str[len-2]) == 'y' &&
-            (tolower(file_str[len-1]) == 'c' ||
-                tolower(file_str[len-1]) == 'o'))
+            unicode[len-4] == '.' &&
+            Py_UNICODE_TOLOWER(unicode[len-3]) == 'p' &&
+            Py_UNICODE_TOLOWER(unicode[len-2]) == 'y' &&
+            (Py_UNICODE_TOLOWER(unicode[len-1]) == 'c' ||
+                Py_UNICODE_TOLOWER(unicode[len-1]) == 'o'))
         {
-            *filename = PyUnicode_FromStringAndSize(file_str, len-1);
-                if (*filename == NULL)
-                        goto handle_error;
-            }
-            else
+            *filename = PyUnicode_FromUnicode(unicode, len-1);
+            if (*filename == NULL)
+                goto handle_error;
+        }
+        else
             Py_INCREF(*filename);
     }
     else {
         const char *module_str = _PyUnicode_AsString(*module);
+        *filename = NULL;
         if (module_str == NULL)
                 goto handle_error;
         if (strcmp(module_str, "__main__") == 0) {
@@ -712,19 +711,17 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds)
 
 
 /* Function to issue a warning message; may raise an exception. */
-int
-PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level)
+
+static int
+warn_unicode(PyObject *category, PyObject *message,
+             Py_ssize_t stack_level)
 {
     PyObject *res;
-    PyObject *message = PyUnicode_FromString(text);
-    if (message == NULL)
-        return -1;
 
     if (category == NULL)
         category = PyExc_RuntimeWarning;
 
     res = do_warn(message, category, stack_level);
-    Py_DECREF(message);
     if (res == NULL)
         return -1;
     Py_DECREF(res);
@@ -732,6 +729,42 @@ PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level)
     return 0;
 }
 
+int
+PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
+                 const char *format, ...)
+{
+    int ret;
+    PyObject *message;
+    va_list vargs;
+
+#ifdef HAVE_STDARG_PROTOTYPES
+    va_start(vargs, format);
+#else
+    va_start(vargs);
+#endif
+    message = PyUnicode_FromFormatV(format, vargs);
+    if (message != NULL) {
+        ret = warn_unicode(category, message, stack_level);
+        Py_DECREF(message);
+    }
+    else
+        ret = -1;
+    va_end(vargs);
+    return ret;
+}
+
+int
+PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level)
+{
+    int ret;
+    PyObject *message = PyUnicode_FromString(text);
+    if (message == NULL)
+        return -1;
+    ret = warn_unicode(category, message, stack_level);
+    Py_DECREF(message);
+    return ret;
+}
+
 /* PyErr_Warn is only for backwards compatibility and will be removed.
    Use PyErr_WarnEx instead. */
 
@@ -751,7 +784,7 @@ PyErr_WarnExplicit(PyObject *category, const char *text,
 {
     PyObject *res;
     PyObject *message = PyUnicode_FromString(text);
-    PyObject *filename = PyUnicode_FromString(filename_str);
+    PyObject *filename = PyUnicode_DecodeFSDefault(filename_str);
     PyObject *module = NULL;
     int ret = -1;
 
@@ -803,6 +836,7 @@ create_filter(PyObject *category, const char *action)
     static PyObject *ignore_str = NULL;
     static PyObject *error_str = NULL;
     static PyObject *default_str = NULL;
+    static PyObject *always_str = NULL;
     PyObject *action_obj = NULL;
     PyObject *lineno, *result;
 
@@ -830,6 +864,14 @@ create_filter(PyObject *category, const char *action)
         }
         action_obj = default_str;
     }
+    else if (!strcmp(action, "always")) {
+        if (always_str == NULL) {
+            always_str = PyUnicode_InternFromString("always");
+            if (always_str == NULL)
+                return NULL;
+        }
+        action_obj = always_str;
+    }
     else {
         Py_FatalError("unknown action");
     }
@@ -846,28 +888,41 @@ create_filter(PyObject *category, const char *action)
 static PyObject *
 init_filters(void)
 {
-    PyObject *filters = PyList_New(3);
-    const char *bytes_action;
+    PyObject *filters = PyList_New(5);
+    unsigned int pos = 0;  /* Post-incremented in each use. */
+    unsigned int x;
+    const char *bytes_action, *resource_action;
+
     if (filters == NULL)
         return NULL;
 
-    PyList_SET_ITEM(filters, 0,
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_DeprecationWarning, "ignore"));
+    PyList_SET_ITEM(filters, pos++,
                     create_filter(PyExc_PendingDeprecationWarning, "ignore"));
-    PyList_SET_ITEM(filters, 1, create_filter(PyExc_ImportWarning, "ignore"));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_ImportWarning, "ignore"));
     if (Py_BytesWarningFlag > 1)
         bytes_action = "error";
     else if (Py_BytesWarningFlag)
         bytes_action = "default";
     else
         bytes_action = "ignore";
-    PyList_SET_ITEM(filters, 2, create_filter(PyExc_BytesWarning,
+    PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning,
                     bytes_action));
-
-    if (PyList_GET_ITEM(filters, 0) == NULL ||
-        PyList_GET_ITEM(filters, 1) == NULL ||
-        PyList_GET_ITEM(filters, 2) == NULL) {
-        Py_DECREF(filters);
-        return NULL;
+    /* resource usage warnings are enabled by default in pydebug mode */
+#ifdef Py_DEBUG
+    resource_action = "always";
+#else
+    resource_action = "ignore";
+#endif
+    PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning,
+                    resource_action));
+    for (x = 0; x < pos; x += 1) {
+        if (PyList_GET_ITEM(filters, x) == NULL) {
+            Py_DECREF(filters);
+            return NULL;
+        }
     }
 
     return filters;
@@ -906,13 +961,13 @@ _PyWarnings_Init(void)
     if (_once_registry == NULL)
         return NULL;
     Py_INCREF(_once_registry);
-    if (PyModule_AddObject(m, "once_registry", _once_registry) < 0)
+    if (PyModule_AddObject(m, "_onceregistry", _once_registry) < 0)
         return NULL;
 
     _default_action = PyUnicode_FromString("default");
     if (_default_action == NULL)
         return NULL;
-    if (PyModule_AddObject(m, "default_action", _default_action) < 0)
+    if (PyModule_AddObject(m, "_defaultaction", _default_action) < 0)
         return NULL;
     return m;
 }