Merge from 3.1: Issue #13703: add a way to randomize the hash values of basic types (str, bytes, datetime)

in order to make algorithmic complexity attacks on (e.g.) web apps much more complicated.

The environment variable PYTHONHASHSEED and the new command line flag -R control this
behavior.

1 files changed, 95 insertions, 21 deletions

diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index af4aa0877e..5054932596 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -16,9 +16,11 @@ extern "C" {
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
 
+#ifndef Py_LIMITED_API
 typedef struct {
     int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
 } PyCompilerFlags;
+#endif
 
 PyAPI_FUNC(void) Py_SetProgramName(wchar_t *);
 PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
@@ -33,41 +35,87 @@ PyAPI_FUNC(int) Py_IsInitialized(void);
 PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
 PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
 
-PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
+#ifndef Py_LIMITED_API
 PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
-
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
-                                                 int, PyCompilerFlags *flags,
-                                                 PyArena *);
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *,
-                                               const char*, int,
-                                               char *, char *,
-                                               PyCompilerFlags *, int *,
-                                               PyArena *);
+PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
+PyAPI_FUNC(int) PyRun_AnyFileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int closeit,
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_SimpleFileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int closeit,
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveOneFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyCompilerFlags *flags);
+
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(
+    const char *s,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyCompilerFlags *flags,
+    PyArena *arena);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    const char* enc,
+    int start,
+    char *ps1,
+    char *ps2,
+    PyCompilerFlags *flags,
+    int *errcode,
+    PyArena *arena);
+#endif
+
+#ifndef PyParser_SimpleParseString
 #define PyParser_SimpleParseString(S, B) \
     PyParser_SimpleParseStringFlags(S, B, 0)
 #define PyParser_SimpleParseFile(FP, S, B) \
     PyParser_SimpleParseFileFlags(FP, S, B, 0)
+#endif
 PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int,
                                                           int);
 PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *,
                                                         int, int);
 
+#ifndef Py_LIMITED_API
 PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *,
                                          PyObject *, PyCompilerFlags *);
 
-PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int,
-                                         PyObject *, PyObject *, int,
-                                         PyCompilerFlags *);
+PyAPI_FUNC(PyObject *) PyRun_FileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyObject *globals,
+    PyObject *locals,
+    int closeit,
+    PyCompilerFlags *flags);
+#endif
 
-#define Py_CompileString(str, p, s) Py_CompileStringFlags(str, p, s, NULL)
-PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int,
-                                             PyCompilerFlags *);
-PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int);
+#ifdef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
+#else
+#define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1)
+#define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1)
+PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(
+    const char *str,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyCompilerFlags *flags,
+    int optimize);
+#endif
+PyAPI_FUNC(struct symtable *) Py_SymtableString(
+    const char *str,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start);
 
 PyAPI_FUNC(void) PyErr_Print(void);
 PyAPI_FUNC(void) PyErr_PrintEx(int);
@@ -76,16 +124,24 @@ PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *);
 /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level
  * exit functions.
  */
+#ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(void));
+#endif
 PyAPI_FUNC(int) Py_AtExit(void (*func)(void));
 
 PyAPI_FUNC(void) Py_Exit(int);
 
+/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _Py_RestoreSignals(void);
+
 PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
+#endif
 
 /* Bootstrap */
 PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
 
+#ifndef Py_LIMITED_API
 /* Use macros for a bunch of old variants */
 #define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL)
 #define PyRun_AnyFile(fp, name) PyRun_AnyFileExFlags(fp, name, 0, NULL)
@@ -104,12 +160,17 @@ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
     PyRun_FileExFlags(fp, p, s, g, l, c, NULL)
 #define PyRun_FileFlags(fp, p, s, g, l, flags) \
     PyRun_FileExFlags(fp, p, s, g, l, 0, flags)
+#endif
 
 /* In getpath.c */
 PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void);
 PyAPI_FUNC(wchar_t *) Py_GetPrefix(void);
 PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void);
 PyAPI_FUNC(wchar_t *) Py_GetPath(void);
+PyAPI_FUNC(void)      Py_SetPath(const wchar_t *);
+#ifdef MS_WINDOWS
+int _Py_CheckPython3();
+#endif
 
 /* In their own files */
 PyAPI_FUNC(const char *) Py_GetVersion(void);
@@ -117,13 +178,16 @@ PyAPI_FUNC(const char *) Py_GetPlatform(void);
 PyAPI_FUNC(const char *) Py_GetCopyright(void);
 PyAPI_FUNC(const char *) Py_GetCompiler(void);
 PyAPI_FUNC(const char *) Py_GetBuildInfo(void);
+#ifndef Py_LIMITED_API
 PyAPI_FUNC(const char *) _Py_svnversion(void);
 PyAPI_FUNC(const char *) Py_SubversionRevision(void);
 PyAPI_FUNC(const char *) Py_SubversionShortBranch(void);
 PyAPI_FUNC(const char *) _Py_hgidentifier(void);
 PyAPI_FUNC(const char *) _Py_hgversion(void);
+#endif
 
 /* Internal -- various one-time initializations */
+#ifndef Py_LIMITED_API
 PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
 PyAPI_FUNC(PyObject *) _PySys_Init(void);
 PyAPI_FUNC(void) _PyImport_Init(void);
@@ -132,8 +196,10 @@ PyAPI_FUNC(void) _PyImportHooks_Init(void);
 PyAPI_FUNC(int) _PyFrame_Init(void);
 PyAPI_FUNC(void) _PyFloat_Init(void);
 PyAPI_FUNC(int) PyByteArray_Init(void);
+#endif
 
 /* Various internal finalizers */
+#ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _PyExc_Fini(void);
 PyAPI_FUNC(void) _PyImport_Fini(void);
 PyAPI_FUNC(void) PyMethod_Fini(void);
@@ -147,12 +213,20 @@ PyAPI_FUNC(void) PyBytes_Fini(void);
 PyAPI_FUNC(void) PyByteArray_Fini(void);
 PyAPI_FUNC(void) PyFloat_Fini(void);
 PyAPI_FUNC(void) PyOS_FiniInterrupts(void);
+PyAPI_FUNC(void) _PyGC_Fini(void);
+
+PyAPI_DATA(PyThreadState *) _Py_Finalizing;
+#endif
 
 /* Stuff with no proper home (yet) */
+#ifndef Py_LIMITED_API
 PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, char *);
+#endif
 PyAPI_DATA(int) (*PyOS_InputHook)(void);
 PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, char *);
+#ifndef Py_LIMITED_API
 PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState;
+#endif
 
 /* Stack size, in "pointers" (so we get extra safety margins
    on 64-bit platforms).  On a 32-bit platform, this translates