From b4220c7f86885b91ebc225da4b74a19e1ed747eb Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Thu, 7 Oct 2010 21:45:39 +0000
Subject: Create fileutils.c/.h

 * _Py_fopen() and _Py_stat() come from Python/import.c
 * (_Py)_wrealpath() comes from Python/sysmodule.c
 * _Py_char2wchar(), _Py_wchar2char() and _Py_wfopen() come from Modules/main.c
 * (_Py)_wstat(), (_Py)_wgetcwd(), _Py_wreadlink() come from Modules/getpath.c
---
 Python/fileutils.c | 758 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 758 insertions(+)
 create mode 100644 Python/fileutils.c

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
new file mode 100644
index 0000000000..ad8b840ef7
--- /dev/null
+++ b/Python/fileutils.c
@@ -0,0 +1,758 @@
+#include "Python.h"
+
+#ifdef HAVE_STAT
+
+/* Decode a byte string from the locale encoding with the
+   surrogateescape error handler (undecodable bytes are decoded as characters
+   in range U+DC80..U+DCFF). If a byte sequence can be decoded as a surrogate
+   character, escape the bytes using the surrogateescape error handler instead
+   of decoding them.
+
+   Use _Py_wchar2char() to encode the character string back to a byte string.
+
+   Return a pointer to a newly allocated (wide) character string (use
+   PyMem_Free() to free the memory), or NULL on error (conversion error or
+   memory error). */
+wchar_t*
+_Py_char2wchar(char* arg)
+{
+    wchar_t *res;
+#ifdef HAVE_BROKEN_MBSTOWCS
+    /* Some platforms have a broken implementation of
+     * mbstowcs which does not count the characters that
+     * would result from conversion.  Use an upper bound.
+     */
+    size_t argsize = strlen(arg);
+#else
+    size_t argsize = mbstowcs(NULL, arg, 0);
+#endif
+    size_t count;
+    unsigned char *in;
+    wchar_t *out;
+#ifdef HAVE_MBRTOWC
+    mbstate_t mbs;
+#endif
+    if (argsize != (size_t)-1) {
+        res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+        if (!res)
+            goto oom;
+        count = mbstowcs(res, arg, argsize+1);
+        if (count != (size_t)-1) {
+            wchar_t *tmp;
+            /* Only use the result if it contains no
+               surrogate characters. */
+            for (tmp = res; *tmp != 0 &&
+                         (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
+                ;
+            if (*tmp == 0)
+                return res;
+        }
+        PyMem_Free(res);
+    }
+    /* Conversion failed. Fall back to escaping with surrogateescape. */
+#ifdef HAVE_MBRTOWC
+    /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
+
+    /* Overallocate; as multi-byte characters are in the argument, the
+       actual output could use less memory. */
+    argsize = strlen(arg) + 1;
+    res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
+    if (!res) goto oom;
+    in = (unsigned char*)arg;
+    out = res;
+    memset(&mbs, 0, sizeof mbs);
+    while (argsize) {
+        size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
+        if (converted == 0)
+            /* Reached end of string; null char stored. */
+            break;
+        if (converted == (size_t)-2) {
+            /* Incomplete character. This should never happen,
+               since we provide everything that we have -
+               unless there is a bug in the C library, or I
+               misunderstood how mbrtowc works. */
+            fprintf(stderr, "unexpected mbrtowc result -2\n");
+            return NULL;
+        }
+        if (converted == (size_t)-1) {
+            /* Conversion error. Escape as UTF-8b, and start over
+               in the initial shift state. */
+            *out++ = 0xdc00 + *in++;
+            argsize--;
+            memset(&mbs, 0, sizeof mbs);
+            continue;
+        }
+        if (*out >= 0xd800 && *out <= 0xdfff) {
+            /* Surrogate character.  Escape the original
+               byte sequence with surrogateescape. */
+            argsize -= converted;
+            while (converted--)
+                *out++ = 0xdc00 + *in++;
+            continue;
+        }
+        /* successfully converted some bytes */
+        in += converted;
+        argsize -= converted;
+        out++;
+    }
+#else
+    /* Cannot use C locale for escaping; manually escape as if charset
+       is ASCII (i.e. escape all bytes > 128. This will still roundtrip
+       correctly in the locale's charset, which must be an ASCII superset. */
+    res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
+    if (!res) goto oom;
+    in = (unsigned char*)arg;
+    out = res;
+    while(*in)
+        if(*in < 128)
+            *out++ = *in++;
+        else
+            *out++ = 0xdc00 + *in++;
+    *out = 0;
+#endif
+    return res;
+oom:
+    fprintf(stderr, "out of memory\n");
+    return NULL;
+}
+
+/* Encode a (wide) character string to the locale encoding with the
+   surrogateescape error handler (characters in range U+DC80..U+DCFF are
+   converted to bytes 0x80..0xFF).
+
+   This function is the reverse of _Py_char2wchar().
+
+   Return a pointer to a newly allocated byte string (use PyMem_Free() to free
+   the memory), or NULL on error (conversion error or memory error). */
+char*
+_Py_wchar2char(const wchar_t *text)
+{
+    const size_t len = wcslen(text);
+    char *result = NULL, *bytes = NULL;
+    size_t i, size, converted;
+    wchar_t c, buf[2];
+
+    /* The function works in two steps:
+       1. compute the length of the output buffer in bytes (size)
+       2. outputs the bytes */
+    size = 0;
+    buf[1] = 0;
+    while (1) {
+        for (i=0; i < len; i++) {
+            c = text[i];
+            if (c >= 0xdc80 && c <= 0xdcff) {
+                /* UTF-8b surrogate */
+                if (bytes != NULL) {
+                    *bytes++ = c - 0xdc00;
+                    size--;
+                }
+                else
+                    size++;
+                continue;
+            }
+            else {
+                buf[0] = c;
+                if (bytes != NULL)
+                    converted = wcstombs(bytes, buf, size);
+                else
+                    converted = wcstombs(NULL, buf, 0);
+                if (converted == (size_t)-1) {
+                    if (result != NULL)
+                        PyMem_Free(result);
+                    return NULL;
+                }
+                if (bytes != NULL) {
+                    bytes += converted;
+                    size -= converted;
+                }
+                else
+                    size += converted;
+            }
+        }
+        if (result != NULL) {
+            *bytes = 0;
+            break;
+        }
+
+        size += 1; /* nul byte at the end */
+        result = PyMem_Malloc(size);
+        if (result == NULL)
+            return NULL;
+        bytes = result;
+    }
+    return result;
+}
+
+#if defined(MS_WINDOWS) || defined(HAVE_STAT)
+int
+_Py_wstat(const wchar_t* path, struct stat *buf)
+{
+/* In principle, this should use HAVE__WSTAT, and _wstat
+   should be detected by autoconf. However, no current
+   POSIX system provides that function, so testing for
+   it is pointless.
+   Not sure whether the MS_WINDOWS guards are necessary:
+   perhaps for cygwin/mingw builds?
+*/
+#ifdef MS_WINDOWS
+    return _wstat(path, buf);
+#else
+    int err;
+    char *fname;
+    fname = _Py_wchar2char(path);
+    if (fname == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    err = stat(fname, buf);
+    PyMem_Free(fname);
+    return err;
+#endif
+}
+#endif
+
+/* Call _wstat() on Windows, or stat() otherwise. Only fill st_mode
+   attribute on Windows. Return 0 on success, -1 on stat error or (if
+   PyErr_Occurred()) unicode error. */
+
+int
+_Py_stat(PyObject *unicode, struct stat *statbuf)
+{
+#ifdef MS_WINDOWS
+    wchar_t *path;
+    int err;
+    struct _stat wstatbuf;
+
+    path = PyUnicode_AsWideCharString(unicode, NULL);
+    if (path == NULL)
+        return -1;
+    err = _wstat(path, &wstatbuf);
+    PyMem_Free(path);
+    if (!err)
+        statbuf->st_mode = wstatbuf.st_mode;
+    return err;
+#else
+    int ret;
+    PyObject *bytes = PyUnicode_EncodeFSDefault(unicode);
+    if (bytes == NULL)
+        return -1;
+    ret = stat(PyBytes_AS_STRING(bytes), statbuf);
+    Py_DECREF(bytes);
+    return ret;
+#endif
+}
+
+FILE *
+_Py_wfopen(const wchar_t *path, const wchar_t *mode)
+{
+#ifndef MS_WINDOWS
+    FILE *f;
+    char *cpath;
+    char cmode[10];
+    size_t r;
+    r = wcstombs(cmode, mode, 10);
+    if (r == (size_t)-1 || r >= 10) {
+        errno = EINVAL;
+        return NULL;
+    }
+    cpath = _Py_wchar2char(path);
+    if (cpath == NULL)
+        return NULL;
+    f = fopen(cpath, cmode);
+    PyMem_Free(cpath);
+    return f;
+#else
+    return _wfopen(path, mode);
+#endif
+}
+
+/* Call _wfopen() on Windows, or fopen() otherwise. Return the new file
+   object on success, or NULL if the file cannot be open or (if
+   PyErr_Occurred()) on unicode error */
+
+FILE*
+_Py_fopen(PyObject *unicode, const char *mode)
+{
+#ifdef MS_WINDOWS
+    wchar_t *path;
+    wchar_t wmode[10];
+    int usize;
+    FILE *f;
+
+    usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, wmode, sizeof(wmode));
+    if (usize == 0)
+        return NULL;
+
+    path = PyUnicode_AsWideCharString(unicode, NULL);
+    if (path == NULL)
+        return NULL;
+    f = _wfopen(path, wmode);
+    PyMem_Free(path);
+    return f;
+#else
+    FILE *f;
+    PyObject *bytes = PyUnicode_EncodeFSDefault(unicode);
+    if (bytes == NULL)
+        return NULL;
+    f = fopen(PyBytes_AS_STRING(bytes), mode);
+    Py_DECREF(bytes);
+    return f;
+#endif
+}
+
+#ifdef HAVE_READLINK
+int
+_Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
+{
+    char *cpath;
+    char cbuf[PATH_MAX];
+    int res;
+    size_t r1;
+
+    cpath = _Py_wchar2char(path);
+    if (cpath == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    res = (int)readlink(cpath, cbuf, PATH_MAX);
+    PyMem_Free(cpath);
+    if (res == -1)
+        return -1;
+    if (res == PATH_MAX) {
+        errno = EINVAL;
+        return -1;
+    }
+    cbuf[res] = '\0'; /* buf will be null terminated */
+    r1 = mbstowcs(buf, cbuf, bufsiz);
+    if (r1 == -1) {
+        errno = EINVAL;
+        return -1;
+    }
+    return (int)r1;
+}
+#endif
+
+#ifdef HAVE_REALPATH
+wchar_t*
+_Py_wrealpath(const wchar_t *path, wchar_t *resolved_path)
+{
+    char *cpath;
+    char cresolved_path[PATH_MAX];
+    char *res;
+    size_t r;
+    cpath = _Py_wchar2char(path);
+    if (cpath == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    res = realpath(cpath, cresolved_path);
+    PyMem_Free(cpath);
+    if (res == NULL)
+        return NULL;
+    r = mbstowcs(resolved_path, cresolved_path, PATH_MAX);
+    if (r == (size_t)-1 || r >= PATH_MAX) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return resolved_path;
+}
+#endif
+
+wchar_t*
+_Py_wgetcwd(wchar_t *buf, size_t size)
+{
+#ifdef MS_WINDOWS
+    return _wgetcwd(buf, size);
+#else
+    char fname[PATH_MAX];
+    if (getcwd(fname, PATH_MAX) == NULL)
+        return NULL;
+    if (mbstowcs(buf, fname, size) >= size) {
+        errno = ERANGE;
+        return NULL;
+    }
+    return buf;
+#endif
+}
+
+#endif
+
+#include "Python.h"
+
+#ifdef HAVE_STAT
+
+/* Decode a byte string from the locale encoding with the
+   surrogateescape error handler (undecodable bytes are decoded as characters
+   in range U+DC80..U+DCFF). If a byte sequence can be decoded as a surrogate
+   character, escape the bytes using the surrogateescape error handler instead
+   of decoding them.
+
+   Use _Py_wchar2char() to encode the character string back to a byte string.
+
+   Return a pointer to a newly allocated (wide) character string (use
+   PyMem_Free() to free the memory), or NULL on error (conversion error or
+   memory error). */
+wchar_t*
+_Py_char2wchar(char* arg)
+{
+    wchar_t *res;
+#ifdef HAVE_BROKEN_MBSTOWCS
+    /* Some platforms have a broken implementation of
+     * mbstowcs which does not count the characters that
+     * would result from conversion.  Use an upper bound.
+     */
+    size_t argsize = strlen(arg);
+#else
+    size_t argsize = mbstowcs(NULL, arg, 0);
+#endif
+    size_t count;
+    unsigned char *in;
+    wchar_t *out;
+#ifdef HAVE_MBRTOWC
+    mbstate_t mbs;
+#endif
+    if (argsize != (size_t)-1) {
+        res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+        if (!res)
+            goto oom;
+        count = mbstowcs(res, arg, argsize+1);
+        if (count != (size_t)-1) {
+            wchar_t *tmp;
+            /* Only use the result if it contains no
+               surrogate characters. */
+            for (tmp = res; *tmp != 0 &&
+                         (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
+                ;
+            if (*tmp == 0)
+                return res;
+        }
+        PyMem_Free(res);
+    }
+    /* Conversion failed. Fall back to escaping with surrogateescape. */
+#ifdef HAVE_MBRTOWC
+    /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
+
+    /* Overallocate; as multi-byte characters are in the argument, the
+       actual output could use less memory. */
+    argsize = strlen(arg) + 1;
+    res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
+    if (!res) goto oom;
+    in = (unsigned char*)arg;
+    out = res;
+    memset(&mbs, 0, sizeof mbs);
+    while (argsize) {
+        size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
+        if (converted == 0)
+            /* Reached end of string; null char stored. */
+            break;
+        if (converted == (size_t)-2) {
+            /* Incomplete character. This should never happen,
+               since we provide everything that we have -
+               unless there is a bug in the C library, or I
+               misunderstood how mbrtowc works. */
+            fprintf(stderr, "unexpected mbrtowc result -2\n");
+            return NULL;
+        }
+        if (converted == (size_t)-1) {
+            /* Conversion error. Escape as UTF-8b, and start over
+               in the initial shift state. */
+            *out++ = 0xdc00 + *in++;
+            argsize--;
+            memset(&mbs, 0, sizeof mbs);
+            continue;
+        }
+        if (*out >= 0xd800 && *out <= 0xdfff) {
+            /* Surrogate character.  Escape the original
+               byte sequence with surrogateescape. */
+            argsize -= converted;
+            while (converted--)
+                *out++ = 0xdc00 + *in++;
+            continue;
+        }
+        /* successfully converted some bytes */
+        in += converted;
+        argsize -= converted;
+        out++;
+    }
+#else
+    /* Cannot use C locale for escaping; manually escape as if charset
+       is ASCII (i.e. escape all bytes > 128. This will still roundtrip
+       correctly in the locale's charset, which must be an ASCII superset. */
+    res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
+    if (!res) goto oom;
+    in = (unsigned char*)arg;
+    out = res;
+    while(*in)
+        if(*in < 128)
+            *out++ = *in++;
+        else
+            *out++ = 0xdc00 + *in++;
+    *out = 0;
+#endif
+    return res;
+oom:
+    fprintf(stderr, "out of memory\n");
+    return NULL;
+}
+
+/* Encode a (wide) character string to the locale encoding with the
+   surrogateescape error handler (characters in range U+DC80..U+DCFF are
+   converted to bytes 0x80..0xFF).
+
+   This function is the reverse of _Py_char2wchar().
+
+   Return a pointer to a newly allocated byte string (use PyMem_Free() to free
+   the memory), or NULL on error (conversion error or memory error). */
+char*
+_Py_wchar2char(const wchar_t *text)
+{
+    const size_t len = wcslen(text);
+    char *result = NULL, *bytes = NULL;
+    size_t i, size, converted;
+    wchar_t c, buf[2];
+
+    /* The function works in two steps:
+       1. compute the length of the output buffer in bytes (size)
+       2. outputs the bytes */
+    size = 0;
+    buf[1] = 0;
+    while (1) {
+        for (i=0; i < len; i++) {
+            c = text[i];
+            if (c >= 0xdc80 && c <= 0xdcff) {
+                /* UTF-8b surrogate */
+                if (bytes != NULL) {
+                    *bytes++ = c - 0xdc00;
+                    size--;
+                }
+                else
+                    size++;
+                continue;
+            }
+            else {
+                buf[0] = c;
+                if (bytes != NULL)
+                    converted = wcstombs(bytes, buf, size);
+                else
+                    converted = wcstombs(NULL, buf, 0);
+                if (converted == (size_t)-1) {
+                    if (result != NULL)
+                        PyMem_Free(result);
+                    return NULL;
+                }
+                if (bytes != NULL) {
+                    bytes += converted;
+                    size -= converted;
+                }
+                else
+                    size += converted;
+            }
+        }
+        if (result != NULL) {
+            *bytes = 0;
+            break;
+        }
+
+        size += 1; /* nul byte at the end */
+        result = PyMem_Malloc(size);
+        if (result == NULL)
+            return NULL;
+        bytes = result;
+    }
+    return result;
+}
+
+#if defined(MS_WINDOWS) || defined(HAVE_STAT)
+int
+_Py_wstat(const wchar_t* path, struct stat *buf)
+{
+/* In principle, this should use HAVE__WSTAT, and _wstat
+   should be detected by autoconf. However, no current
+   POSIX system provides that function, so testing for
+   it is pointless.
+   Not sure whether the MS_WINDOWS guards are necessary:
+   perhaps for cygwin/mingw builds?
+*/
+#ifdef MS_WINDOWS
+    return _wstat(path, buf);
+#else
+    int err;
+    char *fname;
+    fname = _Py_wchar2char(path);
+    if (fname == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    err = stat(fname, buf);
+    PyMem_Free(fname);
+    return err;
+#endif
+}
+#endif
+
+/* Call _wstat() on Windows, or stat() otherwise. Only fill st_mode
+   attribute on Windows. Return 0 on success, -1 on stat error or (if
+   PyErr_Occurred()) unicode error. */
+
+int
+_Py_stat(PyObject *unicode, struct stat *statbuf)
+{
+#ifdef MS_WINDOWS
+    wchar_t *path;
+    int err;
+    struct _stat wstatbuf;
+
+    path = PyUnicode_AsWideCharString(unicode, NULL);
+    if (path == NULL)
+        return -1;
+    err = _wstat(path, &wstatbuf);
+    PyMem_Free(path);
+    if (!err)
+        statbuf->st_mode = wstatbuf.st_mode;
+    return err;
+#else
+    int ret;
+    PyObject *bytes = PyUnicode_EncodeFSDefault(unicode);
+    if (bytes == NULL)
+        return -1;
+    ret = stat(PyBytes_AS_STRING(bytes), statbuf);
+    Py_DECREF(bytes);
+    return ret;
+#endif
+}
+
+FILE *
+_Py_wfopen(const wchar_t *path, const wchar_t *mode)
+{
+#ifndef MS_WINDOWS
+    FILE *f;
+    char *cpath;
+    char cmode[10];
+    size_t r;
+    r = wcstombs(cmode, mode, 10);
+    if (r == (size_t)-1 || r >= 10) {
+        errno = EINVAL;
+        return NULL;
+    }
+    cpath = _Py_wchar2char(path);
+    if (cpath == NULL)
+        return NULL;
+    f = fopen(cpath, cmode);
+    PyMem_Free(cpath);
+    return f;
+#else
+    return _wfopen(path, mode);
+#endif
+}
+
+/* Call _wfopen() on Windows, or fopen() otherwise. Return the new file
+   object on success, or NULL if the file cannot be open or (if
+   PyErr_Occurred()) on unicode error */
+
+FILE*
+_Py_fopen(PyObject *unicode, const char *mode)
+{
+#ifdef MS_WINDOWS
+    wchar_t *path;
+    wchar_t wmode[10];
+    int usize;
+    FILE *f;
+
+    usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, wmode, sizeof(wmode));
+    if (usize == 0)
+        return NULL;
+
+    path = PyUnicode_AsWideCharString(unicode, NULL);
+    if (path == NULL)
+        return NULL;
+    f = _wfopen(path, wmode);
+    PyMem_Free(path);
+    return f;
+#else
+    FILE *f;
+    PyObject *bytes = PyUnicode_EncodeFSDefault(unicode);
+    if (bytes == NULL)
+        return NULL;
+    f = fopen(PyBytes_AS_STRING(bytes), mode);
+    Py_DECREF(bytes);
+    return f;
+#endif
+}
+
+#ifdef HAVE_READLINK
+int
+_Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
+{
+    char *cpath;
+    char cbuf[PATH_MAX];
+    int res;
+    size_t r1;
+
+    cpath = _Py_wchar2char(path);
+    if (cpath == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    res = (int)readlink(cpath, cbuf, PATH_MAX);
+    PyMem_Free(cpath);
+    if (res == -1)
+        return -1;
+    if (res == PATH_MAX) {
+        errno = EINVAL;
+        return -1;
+    }
+    cbuf[res] = '\0'; /* buf will be null terminated */
+    r1 = mbstowcs(buf, cbuf, bufsiz);
+    if (r1 == -1) {
+        errno = EINVAL;
+        return -1;
+    }
+    return (int)r1;
+}
+#endif
+
+#ifdef HAVE_REALPATH
+wchar_t*
+_Py_wrealpath(const wchar_t *path, wchar_t *resolved_path)
+{
+    char *cpath;
+    char cresolved_path[PATH_MAX];
+    char *res;
+    size_t r;
+    cpath = _Py_wchar2char(path);
+    if (cpath == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    res = realpath(cpath, cresolved_path);
+    PyMem_Free(cpath);
+    if (res == NULL)
+        return NULL;
+    r = mbstowcs(resolved_path, cresolved_path, PATH_MAX);
+    if (r == (size_t)-1 || r >= PATH_MAX) {
+        errno = EINVAL;
+        return NULL;
+    }
+    return resolved_path;
+}
+#endif
+
+wchar_t*
+_Py_wgetcwd(wchar_t *buf, size_t size)
+{
+#ifdef MS_WINDOWS
+    return _wgetcwd(buf, size);
+#else
+    char fname[PATH_MAX];
+    if (getcwd(fname, PATH_MAX) == NULL)
+        return NULL;
+    if (mbstowcs(buf, fname, size) >= size) {
+        errno = ERANGE;
+        return NULL;
+    }
+    return buf;
+#endif
+}
+
+#endif
+
-- 
cgit v1.2.1


From 65efe8bb1a2e13924553a2b96df40bd8642128a9 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Thu, 7 Oct 2010 21:55:44 +0000
Subject: Ooops, fileutils.c contains twice the same code

I suppose that I reapplied my local patch creating Python/fileutils.c whereas
the file already existed.
---
 Python/fileutils.c | 380 -----------------------------------------------------
 1 file changed, 380 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index ad8b840ef7..0e87860e52 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -376,383 +376,3 @@ _Py_wgetcwd(wchar_t *buf, size_t size)
 }
 
 #endif
-
-#include "Python.h"
-
-#ifdef HAVE_STAT
-
-/* Decode a byte string from the locale encoding with the
-   surrogateescape error handler (undecodable bytes are decoded as characters
-   in range U+DC80..U+DCFF). If a byte sequence can be decoded as a surrogate
-   character, escape the bytes using the surrogateescape error handler instead
-   of decoding them.
-
-   Use _Py_wchar2char() to encode the character string back to a byte string.
-
-   Return a pointer to a newly allocated (wide) character string (use
-   PyMem_Free() to free the memory), or NULL on error (conversion error or
-   memory error). */
-wchar_t*
-_Py_char2wchar(char* arg)
-{
-    wchar_t *res;
-#ifdef HAVE_BROKEN_MBSTOWCS
-    /* Some platforms have a broken implementation of
-     * mbstowcs which does not count the characters that
-     * would result from conversion.  Use an upper bound.
-     */
-    size_t argsize = strlen(arg);
-#else
-    size_t argsize = mbstowcs(NULL, arg, 0);
-#endif
-    size_t count;
-    unsigned char *in;
-    wchar_t *out;
-#ifdef HAVE_MBRTOWC
-    mbstate_t mbs;
-#endif
-    if (argsize != (size_t)-1) {
-        res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
-        if (!res)
-            goto oom;
-        count = mbstowcs(res, arg, argsize+1);
-        if (count != (size_t)-1) {
-            wchar_t *tmp;
-            /* Only use the result if it contains no
-               surrogate characters. */
-            for (tmp = res; *tmp != 0 &&
-                         (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
-                ;
-            if (*tmp == 0)
-                return res;
-        }
-        PyMem_Free(res);
-    }
-    /* Conversion failed. Fall back to escaping with surrogateescape. */
-#ifdef HAVE_MBRTOWC
-    /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
-
-    /* Overallocate; as multi-byte characters are in the argument, the
-       actual output could use less memory. */
-    argsize = strlen(arg) + 1;
-    res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
-    if (!res) goto oom;
-    in = (unsigned char*)arg;
-    out = res;
-    memset(&mbs, 0, sizeof mbs);
-    while (argsize) {
-        size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
-        if (converted == 0)
-            /* Reached end of string; null char stored. */
-            break;
-        if (converted == (size_t)-2) {
-            /* Incomplete character. This should never happen,
-               since we provide everything that we have -
-               unless there is a bug in the C library, or I
-               misunderstood how mbrtowc works. */
-            fprintf(stderr, "unexpected mbrtowc result -2\n");
-            return NULL;
-        }
-        if (converted == (size_t)-1) {
-            /* Conversion error. Escape as UTF-8b, and start over
-               in the initial shift state. */
-            *out++ = 0xdc00 + *in++;
-            argsize--;
-            memset(&mbs, 0, sizeof mbs);
-            continue;
-        }
-        if (*out >= 0xd800 && *out <= 0xdfff) {
-            /* Surrogate character.  Escape the original
-               byte sequence with surrogateescape. */
-            argsize -= converted;
-            while (converted--)
-                *out++ = 0xdc00 + *in++;
-            continue;
-        }
-        /* successfully converted some bytes */
-        in += converted;
-        argsize -= converted;
-        out++;
-    }
-#else
-    /* Cannot use C locale for escaping; manually escape as if charset
-       is ASCII (i.e. escape all bytes > 128. This will still roundtrip
-       correctly in the locale's charset, which must be an ASCII superset. */
-    res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
-    if (!res) goto oom;
-    in = (unsigned char*)arg;
-    out = res;
-    while(*in)
-        if(*in < 128)
-            *out++ = *in++;
-        else
-            *out++ = 0xdc00 + *in++;
-    *out = 0;
-#endif
-    return res;
-oom:
-    fprintf(stderr, "out of memory\n");
-    return NULL;
-}
-
-/* Encode a (wide) character string to the locale encoding with the
-   surrogateescape error handler (characters in range U+DC80..U+DCFF are
-   converted to bytes 0x80..0xFF).
-
-   This function is the reverse of _Py_char2wchar().
-
-   Return a pointer to a newly allocated byte string (use PyMem_Free() to free
-   the memory), or NULL on error (conversion error or memory error). */
-char*
-_Py_wchar2char(const wchar_t *text)
-{
-    const size_t len = wcslen(text);
-    char *result = NULL, *bytes = NULL;
-    size_t i, size, converted;
-    wchar_t c, buf[2];
-
-    /* The function works in two steps:
-       1. compute the length of the output buffer in bytes (size)
-       2. outputs the bytes */
-    size = 0;
-    buf[1] = 0;
-    while (1) {
-        for (i=0; i < len; i++) {
-            c = text[i];
-            if (c >= 0xdc80 && c <= 0xdcff) {
-                /* UTF-8b surrogate */
-                if (bytes != NULL) {
-                    *bytes++ = c - 0xdc00;
-                    size--;
-                }
-                else
-                    size++;
-                continue;
-            }
-            else {
-                buf[0] = c;
-                if (bytes != NULL)
-                    converted = wcstombs(bytes, buf, size);
-                else
-                    converted = wcstombs(NULL, buf, 0);
-                if (converted == (size_t)-1) {
-                    if (result != NULL)
-                        PyMem_Free(result);
-                    return NULL;
-                }
-                if (bytes != NULL) {
-                    bytes += converted;
-                    size -= converted;
-                }
-                else
-                    size += converted;
-            }
-        }
-        if (result != NULL) {
-            *bytes = 0;
-            break;
-        }
-
-        size += 1; /* nul byte at the end */
-        result = PyMem_Malloc(size);
-        if (result == NULL)
-            return NULL;
-        bytes = result;
-    }
-    return result;
-}
-
-#if defined(MS_WINDOWS) || defined(HAVE_STAT)
-int
-_Py_wstat(const wchar_t* path, struct stat *buf)
-{
-/* In principle, this should use HAVE__WSTAT, and _wstat
-   should be detected by autoconf. However, no current
-   POSIX system provides that function, so testing for
-   it is pointless.
-   Not sure whether the MS_WINDOWS guards are necessary:
-   perhaps for cygwin/mingw builds?
-*/
-#ifdef MS_WINDOWS
-    return _wstat(path, buf);
-#else
-    int err;
-    char *fname;
-    fname = _Py_wchar2char(path);
-    if (fname == NULL) {
-        errno = EINVAL;
-        return -1;
-    }
-    err = stat(fname, buf);
-    PyMem_Free(fname);
-    return err;
-#endif
-}
-#endif
-
-/* Call _wstat() on Windows, or stat() otherwise. Only fill st_mode
-   attribute on Windows. Return 0 on success, -1 on stat error or (if
-   PyErr_Occurred()) unicode error. */
-
-int
-_Py_stat(PyObject *unicode, struct stat *statbuf)
-{
-#ifdef MS_WINDOWS
-    wchar_t *path;
-    int err;
-    struct _stat wstatbuf;
-
-    path = PyUnicode_AsWideCharString(unicode, NULL);
-    if (path == NULL)
-        return -1;
-    err = _wstat(path, &wstatbuf);
-    PyMem_Free(path);
-    if (!err)
-        statbuf->st_mode = wstatbuf.st_mode;
-    return err;
-#else
-    int ret;
-    PyObject *bytes = PyUnicode_EncodeFSDefault(unicode);
-    if (bytes == NULL)
-        return -1;
-    ret = stat(PyBytes_AS_STRING(bytes), statbuf);
-    Py_DECREF(bytes);
-    return ret;
-#endif
-}
-
-FILE *
-_Py_wfopen(const wchar_t *path, const wchar_t *mode)
-{
-#ifndef MS_WINDOWS
-    FILE *f;
-    char *cpath;
-    char cmode[10];
-    size_t r;
-    r = wcstombs(cmode, mode, 10);
-    if (r == (size_t)-1 || r >= 10) {
-        errno = EINVAL;
-        return NULL;
-    }
-    cpath = _Py_wchar2char(path);
-    if (cpath == NULL)
-        return NULL;
-    f = fopen(cpath, cmode);
-    PyMem_Free(cpath);
-    return f;
-#else
-    return _wfopen(path, mode);
-#endif
-}
-
-/* Call _wfopen() on Windows, or fopen() otherwise. Return the new file
-   object on success, or NULL if the file cannot be open or (if
-   PyErr_Occurred()) on unicode error */
-
-FILE*
-_Py_fopen(PyObject *unicode, const char *mode)
-{
-#ifdef MS_WINDOWS
-    wchar_t *path;
-    wchar_t wmode[10];
-    int usize;
-    FILE *f;
-
-    usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, wmode, sizeof(wmode));
-    if (usize == 0)
-        return NULL;
-
-    path = PyUnicode_AsWideCharString(unicode, NULL);
-    if (path == NULL)
-        return NULL;
-    f = _wfopen(path, wmode);
-    PyMem_Free(path);
-    return f;
-#else
-    FILE *f;
-    PyObject *bytes = PyUnicode_EncodeFSDefault(unicode);
-    if (bytes == NULL)
-        return NULL;
-    f = fopen(PyBytes_AS_STRING(bytes), mode);
-    Py_DECREF(bytes);
-    return f;
-#endif
-}
-
-#ifdef HAVE_READLINK
-int
-_Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
-{
-    char *cpath;
-    char cbuf[PATH_MAX];
-    int res;
-    size_t r1;
-
-    cpath = _Py_wchar2char(path);
-    if (cpath == NULL) {
-        errno = EINVAL;
-        return -1;
-    }
-    res = (int)readlink(cpath, cbuf, PATH_MAX);
-    PyMem_Free(cpath);
-    if (res == -1)
-        return -1;
-    if (res == PATH_MAX) {
-        errno = EINVAL;
-        return -1;
-    }
-    cbuf[res] = '\0'; /* buf will be null terminated */
-    r1 = mbstowcs(buf, cbuf, bufsiz);
-    if (r1 == -1) {
-        errno = EINVAL;
-        return -1;
-    }
-    return (int)r1;
-}
-#endif
-
-#ifdef HAVE_REALPATH
-wchar_t*
-_Py_wrealpath(const wchar_t *path, wchar_t *resolved_path)
-{
-    char *cpath;
-    char cresolved_path[PATH_MAX];
-    char *res;
-    size_t r;
-    cpath = _Py_wchar2char(path);
-    if (cpath == NULL) {
-        errno = EINVAL;
-        return NULL;
-    }
-    res = realpath(cpath, cresolved_path);
-    PyMem_Free(cpath);
-    if (res == NULL)
-        return NULL;
-    r = mbstowcs(resolved_path, cresolved_path, PATH_MAX);
-    if (r == (size_t)-1 || r >= PATH_MAX) {
-        errno = EINVAL;
-        return NULL;
-    }
-    return resolved_path;
-}
-#endif
-
-wchar_t*
-_Py_wgetcwd(wchar_t *buf, size_t size)
-{
-#ifdef MS_WINDOWS
-    return _wgetcwd(buf, size);
-#else
-    char fname[PATH_MAX];
-    if (getcwd(fname, PATH_MAX) == NULL)
-        return NULL;
-    if (mbstowcs(buf, fname, size) >= size) {
-        errno = ERANGE;
-        return NULL;
-    }
-    return buf;
-#endif
-}
-
-#endif
-
-- 
cgit v1.2.1


From 0f61f3e424c9ed4231b0337c66877f4be24f641d Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Thu, 7 Oct 2010 22:09:40 +0000
Subject: Fix fileutils for Windows

 * Don't define _Py_wstat() on Windows, Windows has its own _wstat() function
   with a different API (the stat buffer has another type)
 * Include windows.h
---
 Python/fileutils.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 0e87860e52..5d018670e5 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -1,4 +1,7 @@
 #include "Python.h"
+#ifdef MS_WINDOWS
+#  include <windows.h>
+#endif
 
 #ifdef HAVE_STAT
 
@@ -183,10 +186,6 @@ _Py_wchar2char(const wchar_t *text)
     return result;
 }
 
-#if defined(MS_WINDOWS) || defined(HAVE_STAT)
-int
-_Py_wstat(const wchar_t* path, struct stat *buf)
-{
 /* In principle, this should use HAVE__WSTAT, and _wstat
    should be detected by autoconf. However, no current
    POSIX system provides that function, so testing for
@@ -194,9 +193,10 @@ _Py_wstat(const wchar_t* path, struct stat *buf)
    Not sure whether the MS_WINDOWS guards are necessary:
    perhaps for cygwin/mingw builds?
 */
-#ifdef MS_WINDOWS
-    return _wstat(path, buf);
-#else
+#if defined(HAVE_STAT) && !defined(MS_WINDOWS)
+int
+_Py_wstat(const wchar_t* path, struct stat *buf)
+{
     int err;
     char *fname;
     fname = _Py_wchar2char(path);
@@ -207,7 +207,6 @@ _Py_wstat(const wchar_t* path, struct stat *buf)
     err = stat(fname, buf);
     PyMem_Free(fname);
     return err;
-#endif
 }
 #endif
 
-- 
cgit v1.2.1


From 857734b7d26fb28bf7268a1a733155836feaab55 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Thu, 7 Oct 2010 22:23:10 +0000
Subject: _Py_stat() and _Py_fopen(): avoid PyUnicode_AsWideCharString() on
 Windows

On Windows, Py_UNICODE is wchar_t, so we can avoid the expensive Py_UNICODE*
=> wchar_t* conversion.
---
 Python/fileutils.c | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 5d018670e5..bd6ab5d303 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -215,24 +215,19 @@ _Py_wstat(const wchar_t* path, struct stat *buf)
    PyErr_Occurred()) unicode error. */
 
 int
-_Py_stat(PyObject *unicode, struct stat *statbuf)
+_Py_stat(PyObject *path, struct stat *statbuf)
 {
 #ifdef MS_WINDOWS
-    wchar_t *path;
     int err;
     struct _stat wstatbuf;
 
-    path = PyUnicode_AsWideCharString(unicode, NULL);
-    if (path == NULL)
-        return -1;
-    err = _wstat(path, &wstatbuf);
-    PyMem_Free(path);
+    err = _wstat(PyUnicode_AS_UNICODE(path), &wstatbuf);
     if (!err)
         statbuf->st_mode = wstatbuf.st_mode;
     return err;
 #else
     int ret;
-    PyObject *bytes = PyUnicode_EncodeFSDefault(unicode);
+    PyObject *bytes = PyUnicode_EncodeFSDefault(path);
     if (bytes == NULL)
         return -1;
     ret = stat(PyBytes_AS_STRING(bytes), statbuf);
@@ -270,27 +265,20 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode)
    PyErr_Occurred()) on unicode error */
 
 FILE*
-_Py_fopen(PyObject *unicode, const char *mode)
+_Py_fopen(PyObject *path, const char *mode)
 {
 #ifdef MS_WINDOWS
-    wchar_t *path;
     wchar_t wmode[10];
     int usize;
-    FILE *f;
 
     usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, wmode, sizeof(wmode));
     if (usize == 0)
         return NULL;
 
-    path = PyUnicode_AsWideCharString(unicode, NULL);
-    if (path == NULL)
-        return NULL;
-    f = _wfopen(path, wmode);
-    PyMem_Free(path);
-    return f;
+    return _wfopen(PyUnicode_AS_UNICODE(path), wmode);
 #else
     FILE *f;
-    PyObject *bytes = PyUnicode_EncodeFSDefault(unicode);
+    PyObject *bytes = PyUnicode_EncodeFSDefault(path);
     if (bytes == NULL)
         return NULL;
     f = fopen(PyBytes_AS_STRING(bytes), mode);
-- 
cgit v1.2.1


From f99397e5074b70e10303516e11f73083e1b9dd5d Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Thu, 7 Oct 2010 22:29:53 +0000
Subject: _Py_wrealpath() requires the size of the output buffer

---
 Python/fileutils.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index bd6ab5d303..502868f98c 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -321,7 +321,8 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
 
 #ifdef HAVE_REALPATH
 wchar_t*
-_Py_wrealpath(const wchar_t *path, wchar_t *resolved_path)
+_Py_wrealpath(const wchar_t *path,
+              wchar_t *resolved_path, size_t resolved_path_size)
 {
     char *cpath;
     char cresolved_path[PATH_MAX];
@@ -336,7 +337,7 @@ _Py_wrealpath(const wchar_t *path, wchar_t *resolved_path)
     PyMem_Free(cpath);
     if (res == NULL)
         return NULL;
-    r = mbstowcs(resolved_path, cresolved_path, PATH_MAX);
+    r = mbstowcs(resolved_path, cresolved_path, resolved_path_size);
     if (r == (size_t)-1 || r >= PATH_MAX) {
         errno = EINVAL;
         return NULL;
-- 
cgit v1.2.1


From 8c367daeb71da022f5b85b9029eb0e01cd4b4ced Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Thu, 7 Oct 2010 22:53:43 +0000
Subject: fileutils.c: document which encodings are used

---
 Python/fileutils.c | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 502868f98c..9423cb02f1 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -194,6 +194,9 @@ _Py_wchar2char(const wchar_t *text)
    perhaps for cygwin/mingw builds?
 */
 #if defined(HAVE_STAT) && !defined(MS_WINDOWS)
+
+/* Get file status. Encode the path to the locale encoding. */
+
 int
 _Py_wstat(const wchar_t* path, struct stat *buf)
 {
@@ -210,9 +213,11 @@ _Py_wstat(const wchar_t* path, struct stat *buf)
 }
 #endif
 
-/* Call _wstat() on Windows, or stat() otherwise. Only fill st_mode
-   attribute on Windows. Return 0 on success, -1 on stat error or (if
-   PyErr_Occurred()) unicode error. */
+/* Call _wstat() on Windows, or encode the path to the filesystem encoding and
+   call stat() otherwise. Only fill st_mode attribute on Windows.
+
+   Return 0 on success, -1 on _wstat() / stat() error or (if PyErr_Occurred())
+   unicode error. */
 
 int
 _Py_stat(PyObject *path, struct stat *statbuf)
@@ -236,6 +241,9 @@ _Py_stat(PyObject *path, struct stat *statbuf)
 #endif
 }
 
+/* Open a file. Use _wfopen() on Windows, encode the path to the locale
+   encoding and use fopen() otherwise. */
+
 FILE *
 _Py_wfopen(const wchar_t *path, const wchar_t *mode)
 {
@@ -260,9 +268,11 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode)
 #endif
 }
 
-/* Call _wfopen() on Windows, or fopen() otherwise. Return the new file
-   object on success, or NULL if the file cannot be open or (if
-   PyErr_Occurred()) on unicode error */
+/* Call _wfopen() on Windows, or encode the path to the filesystem encoding and
+   call fopen() otherwise.
+
+   Return the new file object on success, or NULL if the file cannot be open or
+   (if PyErr_Occurred()) on unicode error */
 
 FILE*
 _Py_fopen(PyObject *path, const char *mode)
@@ -288,6 +298,10 @@ _Py_fopen(PyObject *path, const char *mode)
 }
 
 #ifdef HAVE_READLINK
+
+/* Read value of symbolic link. Encode the path to the locale encoding, decode
+   the result from the locale encoding. */
+
 int
 _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
 {
@@ -320,6 +334,10 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
 #endif
 
 #ifdef HAVE_REALPATH
+
+/* Return the canonicalized absolute pathname. Encode path to the locale
+   encoding, decode the result from the locale encoding. */
+
 wchar_t*
 _Py_wrealpath(const wchar_t *path,
               wchar_t *resolved_path, size_t resolved_path_size)
@@ -346,6 +364,8 @@ _Py_wrealpath(const wchar_t *path,
 }
 #endif
 
+/* Get the current directory. Decode the path from the locale encoding. */
+
 wchar_t*
 _Py_wgetcwd(wchar_t *buf, size_t size)
 {
-- 
cgit v1.2.1


From f371f4fe2a7e4e204e69502cb23fce7963bb0296 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Thu, 14 Oct 2010 12:37:19 +0000
Subject: _Py_wgetcwd() decodes the path using _Py_char2wchar() to support
 surrogates

---
 Python/fileutils.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 9423cb02f1..564e2c085b 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -364,7 +364,8 @@ _Py_wrealpath(const wchar_t *path,
 }
 #endif
 
-/* Get the current directory. Decode the path from the locale encoding. */
+/* Get the current directory. size is the buffer size in wide characters
+   including the null character. Decode the path from the locale encoding. */
 
 wchar_t*
 _Py_wgetcwd(wchar_t *buf, size_t size)
@@ -373,12 +374,19 @@ _Py_wgetcwd(wchar_t *buf, size_t size)
     return _wgetcwd(buf, size);
 #else
     char fname[PATH_MAX];
+    wchar_t *wname;
+
     if (getcwd(fname, PATH_MAX) == NULL)
         return NULL;
-    if (mbstowcs(buf, fname, size) >= size) {
-        errno = ERANGE;
+    wname = _Py_char2wchar(fname);
+    if (wname == NULL)
+        return NULL;
+    if (size <= wcslen(wname)) {
+        PyMem_Free(wname);
         return NULL;
     }
+    wcsncpy(buf, wname, size);
+    PyMem_Free(wname);
     return buf;
 #endif
 }
-- 
cgit v1.2.1


From a22d3119013a15abc89aa99a47eaa0e168feb857 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Fri, 15 Oct 2010 11:15:54 +0000
Subject: Mark _Py_char2wchar() input argument as constant

---
 Python/fileutils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 564e2c085b..076f510deb 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -17,7 +17,7 @@
    PyMem_Free() to free the memory), or NULL on error (conversion error or
    memory error). */
 wchar_t*
-_Py_char2wchar(char* arg)
+_Py_char2wchar(const char* arg)
 {
     wchar_t *res;
 #ifdef HAVE_BROKEN_MBSTOWCS
-- 
cgit v1.2.1


From f086369a465ab84e07afcb703c0c130189c0f863 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Sat, 16 Oct 2010 22:47:37 +0000
Subject: _Py_wreadlink() uses _Py_char2wchar() to decode the result, to
 support surrogate characters.

---
 Python/fileutils.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 076f510deb..cfafd865c5 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -307,6 +307,7 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
 {
     char *cpath;
     char cbuf[PATH_MAX];
+    wchar_t *wbuf;
     int res;
     size_t r1;
 
@@ -324,11 +325,15 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
         return -1;
     }
     cbuf[res] = '\0'; /* buf will be null terminated */
-    r1 = mbstowcs(buf, cbuf, bufsiz);
-    if (r1 == -1) {
+    wbuf = _Py_char2wchar(cbuf);
+    r1 = wcslen(wbuf);
+    if (bufsiz <= r1) {
+        PyMem_Free(wbuf);
         errno = EINVAL;
         return -1;
     }
+    wcsncpy(buf, wbuf, bufsiz);
+    PyMem_Free(wbuf);
     return (int)r1;
 }
 #endif
-- 
cgit v1.2.1


From 560612a9ad4dc53b764e2e0cc1137c7687f1d2c3 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Sat, 16 Oct 2010 22:52:09 +0000
Subject: _Py_wreadlink(): catch _Py_char2wchar() failure

---
 Python/fileutils.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index cfafd865c5..147636f2c7 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -326,6 +326,10 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
     }
     cbuf[res] = '\0'; /* buf will be null terminated */
     wbuf = _Py_char2wchar(cbuf);
+    if (wbuf == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
     r1 = wcslen(wbuf);
     if (bufsiz <= r1) {
         PyMem_Free(wbuf);
-- 
cgit v1.2.1


From 4e7ceabd43ad27956ac0014ce5ff16995fd5c03d Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Sat, 16 Oct 2010 22:55:47 +0000
Subject: _Py_wrealpath() uses _Py_char2wchar() to decode the result, to
 support surrogate characters.

---
 Python/fileutils.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 147636f2c7..b8910b7be6 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -353,6 +353,7 @@ _Py_wrealpath(const wchar_t *path,
 {
     char *cpath;
     char cresolved_path[PATH_MAX];
+    wchar_t *wresolved_path;
     char *res;
     size_t r;
     cpath = _Py_wchar2char(path);
@@ -364,11 +365,20 @@ _Py_wrealpath(const wchar_t *path,
     PyMem_Free(cpath);
     if (res == NULL)
         return NULL;
-    r = mbstowcs(resolved_path, cresolved_path, resolved_path_size);
-    if (r == (size_t)-1 || r >= PATH_MAX) {
+
+    wresolved_path = _Py_char2wchar(cresolved_path);
+    if (wresolved_path == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    r = wcslen(wresolved_path);
+    if (resolved_path_size <= r) {
+        PyMem_Free(wresolved_path);
         errno = EINVAL;
         return NULL;
     }
+    wcsncpy(resolved_path, wresolved_path, resolved_path_size);
+    PyMem_Free(wresolved_path);
     return resolved_path;
 }
 #endif
-- 
cgit v1.2.1


From bd3b71a7bb5fcf80de1339fe71c85918a44fb7e3 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Sat, 16 Oct 2010 23:16:16 +0000
Subject: Add an optional size argument to _Py_char2wchar()

_Py_char2wchar() callers usually need the result size in characters. Since it's
trivial to compute it in _Py_char2wchar() (O(1) whereas wcslen() is O(n)), add
an option to get it.
---
 Python/fileutils.c | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index b8910b7be6..03fc0cb79d 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -13,11 +13,12 @@
 
    Use _Py_wchar2char() to encode the character string back to a byte string.
 
-   Return a pointer to a newly allocated (wide) character string (use
-   PyMem_Free() to free the memory), or NULL on error (conversion error or
-   memory error). */
+   Return a pointer to a newly allocated wide character string (use
+   PyMem_Free() to free the memory) and write the number of written wide
+   characters excluding the null character into *size if size is not NULL, or
+   NULL on error (conversion error or memory error). */
 wchar_t*
-_Py_char2wchar(const char* arg)
+_Py_char2wchar(const char* arg, size_t *size)
 {
     wchar_t *res;
 #ifdef HAVE_BROKEN_MBSTOWCS
@@ -47,8 +48,11 @@ _Py_char2wchar(const char* arg)
             for (tmp = res; *tmp != 0 &&
                          (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
                 ;
-            if (*tmp == 0)
+            if (*tmp == 0) {
+                if (size != NULL)
+                    *size = count;
                 return res;
+            }
         }
         PyMem_Free(res);
     }
@@ -113,6 +117,8 @@ _Py_char2wchar(const char* arg)
             *out++ = 0xdc00 + *in++;
     *out = 0;
 #endif
+    if (size != NULL)
+        *size = out - res;
     return res;
 oom:
     fprintf(stderr, "out of memory\n");
@@ -325,12 +331,11 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
         return -1;
     }
     cbuf[res] = '\0'; /* buf will be null terminated */
-    wbuf = _Py_char2wchar(cbuf);
+    wbuf = _Py_char2wchar(cbuf, &r1);
     if (wbuf == NULL) {
         errno = EINVAL;
         return -1;
     }
-    r1 = wcslen(wbuf);
     if (bufsiz <= r1) {
         PyMem_Free(wbuf);
         errno = EINVAL;
@@ -366,12 +371,11 @@ _Py_wrealpath(const wchar_t *path,
     if (res == NULL)
         return NULL;
 
-    wresolved_path = _Py_char2wchar(cresolved_path);
+    wresolved_path = _Py_char2wchar(cresolved_path, &r);
     if (wresolved_path == NULL) {
         errno = EINVAL;
         return NULL;
     }
-    r = wcslen(wresolved_path);
     if (resolved_path_size <= r) {
         PyMem_Free(wresolved_path);
         errno = EINVAL;
@@ -394,13 +398,14 @@ _Py_wgetcwd(wchar_t *buf, size_t size)
 #else
     char fname[PATH_MAX];
     wchar_t *wname;
+    size_t len;
 
     if (getcwd(fname, PATH_MAX) == NULL)
         return NULL;
-    wname = _Py_char2wchar(fname);
+    wname = _Py_char2wchar(fname, &len);
     if (wname == NULL)
         return NULL;
-    if (size <= wcslen(wname)) {
+    if (size <= len) {
         PyMem_Free(wname);
         return NULL;
     }
-- 
cgit v1.2.1


From 004b7b95f3ffafabb761f4d3d6d17b67d11e3119 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Mon, 8 Nov 2010 22:43:46 +0000
Subject: PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails

 * Add error_pos optional argument to _Py_wchar2char()
 * PyUnicode_EncodeFS() raises a UnicodeEncodeError or MemoryError if
   _Py_wchar2char() fails
---
 Python/fileutils.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 03fc0cb79d..18e98e513c 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -132,15 +132,21 @@ oom:
    This function is the reverse of _Py_char2wchar().
 
    Return a pointer to a newly allocated byte string (use PyMem_Free() to free
-   the memory), or NULL on error (conversion error or memory error). */
+   the memory), or NULL on conversion or memory allocation error.
+
+   If error_pos is not NULL: *error_pos is the index of the invalid character
+   on conversion error, or (size_t)-1 otherwise. */
 char*
-_Py_wchar2char(const wchar_t *text)
+_Py_wchar2char(const wchar_t *text, size_t *error_pos)
 {
     const size_t len = wcslen(text);
     char *result = NULL, *bytes = NULL;
     size_t i, size, converted;
     wchar_t c, buf[2];
 
+    if (error_pos != NULL)
+        *error_pos = (size_t)-1;
+
     /* The function works in two steps:
        1. compute the length of the output buffer in bytes (size)
        2. outputs the bytes */
@@ -168,6 +174,8 @@ _Py_wchar2char(const wchar_t *text)
                 if (converted == (size_t)-1) {
                     if (result != NULL)
                         PyMem_Free(result);
+                    if (error_pos != NULL)
+                        *error_pos = i;
                     return NULL;
                 }
                 if (bytes != NULL) {
@@ -208,7 +216,7 @@ _Py_wstat(const wchar_t* path, struct stat *buf)
 {
     int err;
     char *fname;
-    fname = _Py_wchar2char(path);
+    fname = _Py_wchar2char(path, NULL);
     if (fname == NULL) {
         errno = EINVAL;
         return -1;
@@ -263,7 +271,7 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode)
         errno = EINVAL;
         return NULL;
     }
-    cpath = _Py_wchar2char(path);
+    cpath = _Py_wchar2char(path, NULL);
     if (cpath == NULL)
         return NULL;
     f = fopen(cpath, cmode);
@@ -317,7 +325,7 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
     int res;
     size_t r1;
 
-    cpath = _Py_wchar2char(path);
+    cpath = _Py_wchar2char(path, NULL);
     if (cpath == NULL) {
         errno = EINVAL;
         return -1;
@@ -361,7 +369,7 @@ _Py_wrealpath(const wchar_t *path,
     wchar_t *wresolved_path;
     char *res;
     size_t r;
-    cpath = _Py_wchar2char(path);
+    cpath = _Py_wchar2char(path, NULL);
     if (cpath == NULL) {
         errno = EINVAL;
         return NULL;
-- 
cgit v1.2.1


From c1045d4f1ecd8cf5dc79d227214e371f3d6f7903 Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@haypocalc.com>
Date: Mon, 8 Nov 2010 23:30:46 +0000
Subject: _Py_char2wchar() frees the memory on conversion error

Explain in the documentation that conversion errors should never happen.
---
 Python/fileutils.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'Python/fileutils.c')

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 18e98e513c..c563eaa5fb 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -16,7 +16,10 @@
    Return a pointer to a newly allocated wide character string (use
    PyMem_Free() to free the memory) and write the number of written wide
    characters excluding the null character into *size if size is not NULL, or
-   NULL on error (conversion error or memory error). */
+   NULL on error (conversion or memory allocation error).
+
+   Conversion errors should never happen, unless there is a bug in the C
+   library. */
 wchar_t*
 _Py_char2wchar(const char* arg, size_t *size)
 {
@@ -64,7 +67,8 @@ _Py_char2wchar(const char* arg, size_t *size)
        actual output could use less memory. */
     argsize = strlen(arg) + 1;
     res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
-    if (!res) goto oom;
+    if (!res)
+        goto oom;
     in = (unsigned char*)arg;
     out = res;
     memset(&mbs, 0, sizeof mbs);
@@ -79,6 +83,7 @@ _Py_char2wchar(const char* arg, size_t *size)
                unless there is a bug in the C library, or I
                misunderstood how mbrtowc works. */
             fprintf(stderr, "unexpected mbrtowc result -2\n");
+            PyMem_Free(res);
             return NULL;
         }
         if (converted == (size_t)-1) {
-- 
cgit v1.2.1