From 9e9b211f22e4130b886188ca6252ffdedfa55ef5 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 10 Dec 2021 23:35:04 +0100 Subject: MDEV-27089 Windows : incorrect handling of non-ASCIIs in get_tty_password Prior to patch, get_password would echo multple mask characters '*', for a single multibyte input character. Fixed the behavior by using "wide" version of getch, _getwch. Also take care of possible characters outside of BMP (i.e we do not print '*' for high surrogates). The function will now internally construct the "wide" password string, and conver to the console codepage. Some characters could still be lost in that conversion, unless the codepage is utf8, but this is not any new bug. --- mysys/get_password.c | 55 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/mysys/get_password.c b/mysys/get_password.c index 24befa6b5df..bdd20d0349b 100644 --- a/mysys/get_password.c +++ b/mysys/get_password.c @@ -62,35 +62,58 @@ char *get_tty_password(const char *opt_message) { - char to[80]; - char *pos=to,*end=to+sizeof(to)-1; + wchar_t wbuf[80]; + char *to; + int to_len; + UINT cp; + wchar_t *pos=wbuf,*end=wbuf + array_elements(wbuf)-1; DBUG_ENTER("get_tty_password"); _cputs(opt_message ? opt_message : "Enter password: "); for (;;) { - char tmp; - tmp=_getch(); - if (tmp == '\b' || (int) tmp == 127) + int wc; + wc=_getwch(); + if (wc == '\b' || wc == 127) { - if (pos != to) + if (pos != wbuf) { - _cputs("\b \b"); - pos--; - continue; + _cputs("\b \b"); + pos--; + continue; } } - if (tmp == '\n' || tmp == '\r' || tmp == 3) + if (wc == '\n' || wc == '\r' || wc == 3 || pos == end) break; - if (iscntrl(tmp) || pos == end) + if (iswcntrl(wc)) continue; - _cputs("*"); - *(pos++) = tmp; + + /* Do not print '*' for half-unicode char(high surrogate)*/ + if (wc < 0xD800 || wc > 0xDBFF) + { + _cputs("*"); + } + *(pos++)= (wchar_t)wc; } - while (pos != to && isspace(pos[-1]) == ' ') - pos--; /* Allow dummy space at end */ *pos=0; _cputs("\n"); - DBUG_RETURN(my_strdup(PSI_INSTRUMENT_ME, to,MYF(MY_FAE))); + + /* + Allocate output string, and convert UTF16 password to output codepage. + */ + cp= GetConsoleCP(); + + if (!(to_len= WideCharToMultiByte(cp, 0, wbuf, -1, NULL, 0, NULL, NULL))) + DBUG_RETURN(NULL); + + if (!(to= my_malloc(PSI_INSTRUMENT_ME, to_len, MYF(MY_FAE)))) + DBUG_RETURN(NULL); + + if (!WideCharToMultiByte(cp, 0, wbuf, -1, to, to_len, NULL, NULL)) + { + my_free(to); + DBUG_RETURN(NULL); + } + DBUG_RETURN(to); } #else -- cgit v1.2.1 From 99e5ae3b1a68ad1029f06aa3d2a5078d78cf19e5 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 19 Nov 2021 12:03:48 +0100 Subject: MDEV-27090 Windows client - ReadConsoleA does not work correctly with UTF8 codepage Corresponding Windows bug https://github.com/microsoft/terminal/issues/4551 Use ReadConsoleW instead and convert to console's input codepage, to workaround. Also, disable VT sequences in the console output, as we do not knows what type of data comes with SELECT, we do not want VT escapes there. Remove my_cgets() --- client/mysql.cc | 128 +++++++++++++++++++++-------- include/my_sys.h | 7 -- mysys/CMakeLists.txt | 1 - mysys/my_conio.c | 223 --------------------------------------------------- 4 files changed, 95 insertions(+), 264 deletions(-) delete mode 100644 mysys/my_conio.c diff --git a/client/mysql.cc b/client/mysql.cc index ee963b9220d..01a511b35b9 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -88,9 +88,7 @@ extern "C" { #endif /* defined(HAVE_CURSES_H) && defined(HAVE_TERM_H) */ #undef bcmp // Fix problem with new readline -#if defined(_WIN32) -#include -#else +#if !defined(_WIN32) # ifdef __APPLE__ # include # else @@ -104,6 +102,98 @@ extern "C" { #endif } +static CHARSET_INFO *charset_info= &my_charset_latin1; + +#if defined(_WIN32) +/* + Set console mode for the whole duration of the client session. + + We need for input + - line input (i.e read lines from console) + - echo typed characters + - "cooked" mode, i.e we do not want to handle all keystrokes, + like DEL etc ourselves, yet. We might want handle keystrokes + in the future, to implement tab completion, and better + (multiline) history. + + Disable VT escapes for the output.We do not know what kind of escapes SELECT would return. +*/ +struct Console_mode +{ + HANDLE in= GetStdHandle(STD_INPUT_HANDLE); + HANDLE out= GetStdHandle(STD_OUTPUT_HANDLE); + DWORD mode_in=0; + DWORD mode_out=0; + + enum {STDIN_CHANGED = 1, STDOUT_CHANGED = 2}; + int changes=0; + + Console_mode() + { + if (in && in != INVALID_HANDLE_VALUE && GetConsoleMode(in, &mode_in)) + { + SetConsoleMode(in, ENABLE_ECHO_INPUT|ENABLE_LINE_INPUT|ENABLE_PROCESSED_INPUT); + changes |= STDIN_CHANGED; + } + + if (out && out != INVALID_HANDLE_VALUE && GetConsoleMode(out, &mode_out)) + { +#ifdef ENABLE_VIRTUAL_TERMINAL_INPUT + SetConsoleMode(out, mode_out & ~ENABLE_VIRTUAL_TERMINAL_INPUT); + changes |= STDOUT_CHANGED; +#endif + } + } + + ~Console_mode() + { + if (changes & STDIN_CHANGED) + SetConsoleMode(in, mode_in); + + if(changes & STDOUT_CHANGED) + SetConsoleMode(out, mode_out); + } +}; + +static Console_mode my_conmode; + +#define MAX_CGETS_LINE_LEN 65535 +/** Read line from console, chomp EOL*/ +static char *win_readline() +{ + static wchar_t wstrbuf[MAX_CGETS_LINE_LEN]; + static char strbuf[MAX_CGETS_LINE_LEN * 4]; + + DWORD nchars= 0; + uint len= 0; + SetLastError(0); + if (!ReadConsoleW(GetStdHandle(STD_INPUT_HANDLE), wstrbuf, MAX_CGETS_LINE_LEN-1, + &nchars, NULL)) + goto err; + if (nchars == 0 && GetLastError() == ERROR_OPERATION_ABORTED) + goto err; + + for (;nchars > 0; nchars--) + { + if (wstrbuf[nchars - 1] != '\n' && wstrbuf[nchars - 1] != '\r') + break; + } + + if (nchars > 0) + { + uint errors; + len= my_convert(strbuf, sizeof(strbuf), charset_info, + (const char *) wstrbuf, nchars * sizeof(wchar_t), + &my_charset_utf16le_bin, &errors); + } + strbuf[len]= 0; + return strbuf; +err: + return NULL; +} +#endif + + #ifdef HAVE_VIDATTR static int have_curses= 0; static void my_vidattr(chtype attrs) @@ -208,7 +298,6 @@ unsigned short terminal_width= 80; static uint opt_protocol=0; static const char *opt_protocol_type= ""; -static CHARSET_INFO *charset_info= &my_charset_latin1; static uint protocol_to_force= MYSQL_PROTOCOL_DEFAULT; @@ -2033,11 +2122,6 @@ static int get_options(int argc, char **argv) static int read_and_execute(bool interactive) { -#if defined(_WIN32) - String tmpbuf; - String buffer; -#endif - char *line= NULL; char in_string=0; ulong line_number=0; @@ -2115,26 +2199,7 @@ static int read_and_execute(bool interactive) #if defined(_WIN32) tee_fputs(prompt, stdout); - if (!tmpbuf.is_alloced()) - tmpbuf.alloc(65535); - tmpbuf.length(0); - buffer.length(0); - size_t clen; - do - { - line= my_cgets((char*)tmpbuf.ptr(), tmpbuf.alloced_length()-1, &clen); - buffer.append(line, clen); - /* - if we got buffer fully filled than there is a chance that - something else is still in console input buffer - */ - } while (tmpbuf.alloced_length() <= clen); - /* - An empty line is returned from my_cgets when there's error reading : - Ctrl-c for example - */ - if (line) - line= buffer.c_ptr(); + line= win_readline(); #else if (opt_outfile) fputs(prompt, OUTFILE); @@ -2201,10 +2266,7 @@ static int read_and_execute(bool interactive) } } -#if defined(_WIN32) - buffer.free(); - tmpbuf.free(); -#else +#if !defined(_WIN32) if (interactive) /* free the last entered line. diff --git a/include/my_sys.h b/include/my_sys.h index c6526e581cd..5a4608155e4 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -1097,13 +1097,6 @@ extern void thd_increment_bytes_sent(void *thd, size_t length); extern void thd_increment_bytes_received(void *thd, size_t length); extern void thd_increment_net_big_packet_count(void *thd, size_t length); -#ifdef _WIN32 - -/* implemented in my_conio.c */ -char* my_cgets(char *string, size_t clen, size_t* plen); - -#endif - #include #ifdef HAVE_PSI_INTERFACE diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 760c3c1475d..758243df10f 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -54,7 +54,6 @@ IF (WIN32) my_wincond.c my_winerr.c my_winfile.c - my_conio.c my_minidump.cc my_win_popen.cc) ENDIF() diff --git a/mysys/my_conio.c b/mysys/my_conio.c deleted file mode 100644 index abcfa9798ec..00000000000 --- a/mysys/my_conio.c +++ /dev/null @@ -1,223 +0,0 @@ -/* Copyright (c) 2000, 2005, 2007 MySQL AB - Use is subject to license terms - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ - - -#include "mysys_priv.h" - -#ifdef _WIN32 - -static HANDLE my_coninpfh= 0; /* console input */ - -/* - functions my_pthread_auto_mutex_lock & my_pthread_auto_mutex_free - are experimental at this moment, they are intended to bring - ability of protecting code sections without necessity to explicitly - initialize synchronization object in one of threads - - if found useful they are to be exported in mysys -*/ - - -/* - int my_pthread_auto_mutex_lock(HANDLE* ph, const char* name, - int id, int time) - NOTES - creates a mutex with given name and tries to lock it time msec. - mutex name is appended with id to allow system wide or process wide - locks. Handle to created mutex returned in ph argument. - - RETURN - 0 thread owns mutex - <>0 error -*/ - -static -int my_pthread_auto_mutex_lock(HANDLE* ph, const char* name, int id, int time) -{ - DWORD res; - char tname[FN_REFLEN]; - - sprintf(tname, "%s-%08X", name, id); - - *ph= CreateMutex(NULL, FALSE, tname); - if (*ph == NULL) - return GetLastError(); - - res= WaitForSingleObject(*ph, time); - - if (res == WAIT_TIMEOUT) - return ERROR_SEM_TIMEOUT; - - if (res == WAIT_FAILED) - return GetLastError(); - - return 0; -} - -/* - int my_pthread_auto_mutex_free(HANDLE* ph) - - NOTES - releases a mutex. - - RETURN - 0 thread released mutex - <>0 error - -*/ -static -int my_pthread_auto_mutex_free(HANDLE* ph) -{ - if (*ph) - { - ReleaseMutex(*ph); - CloseHandle(*ph); - *ph= NULL; - } - - return 0; -} - - -#define pthread_auto_mutex_decl(name) \ - HANDLE __h##name= NULL; - -#define pthread_auto_mutex_lock(name, proc, time) \ - my_pthread_auto_mutex_lock(&__h##name, #name, (proc), (time)) - -#define pthread_auto_mutex_free(name) \ - my_pthread_auto_mutex_free(&__h##name) - - -/* - char* my_cgets() - - NOTES - Replaces _cgets from libc to support input of more than 255 chars. - Reads from the console via ReadConsole into buffer which - should be at least clen characters. - Actual length of string returned in plen. - - WARNING - my_cgets() does NOT check the pushback character buffer (i.e., _chbuf). - Thus, my_cgets() will not return any character that is pushed back by - the _ungetch() call. - - RETURN - string pointer ok - NULL Error - -*/ - -char* my_cgets(char *buffer, size_t clen, size_t* plen) -{ - ULONG state; - char *result; - DWORD plen_res; - CONSOLE_SCREEN_BUFFER_INFO csbi; - - pthread_auto_mutex_decl(my_conio_cs); - - /* lock the console for the current process*/ - if (pthread_auto_mutex_lock(my_conio_cs, GetCurrentProcessId(), INFINITE)) - { - /* can not lock console */ - pthread_auto_mutex_free(my_conio_cs); - return NULL; - } - - /* init console input */ - if (my_coninpfh == 0) - { - /* same handle will be used until process termination */ - my_coninpfh= CreateFile("CONIN$", GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, - NULL, OPEN_EXISTING, 0, NULL); - } - - if (my_coninpfh == INVALID_HANDLE_VALUE) - { - /* unlock the console */ - pthread_auto_mutex_free(my_conio_cs); - return(NULL); - } - - GetConsoleMode((HANDLE)my_coninpfh, &state); - SetConsoleMode((HANDLE)my_coninpfh, ENABLE_LINE_INPUT | - ENABLE_PROCESSED_INPUT | ENABLE_ECHO_INPUT); - - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi); - - /* - there is no known way to determine allowed buffer size for input - though it is known it should not be more than 64K - so we cut 64K and try first size of screen buffer - if it is still to large we cut half of it and try again - later we may want to cycle from MY_MIN(clen, 65535) to allowed size - with small decrement to determine exact allowed buffer - */ - clen= MY_MIN(clen, 65535); - do - { - clen= MY_MIN(clen, (size_t) csbi.dwSize.X*csbi.dwSize.Y); - if (!ReadConsole((HANDLE)my_coninpfh, (LPVOID)buffer, (DWORD) clen - 1, &plen_res, - NULL)) - { - result= NULL; - clen>>= 1; - } - else - { - result= buffer; - break; - } - } - while (GetLastError() == ERROR_NOT_ENOUGH_MEMORY); - *plen= plen_res; - - /* We go here on error reading the string (Ctrl-C for example) */ - if (!*plen) - result= NULL; /* purecov: inspected */ - - if (result != NULL) - { - if (*plen > 1 && buffer[*plen - 2] == '\r') - { - *plen= *plen - 2; - } - else - { - if (*plen > 0 && buffer[*plen - 1] == '\r') - { - char tmp[3]; - DWORD tmplen= (DWORD)sizeof(tmp); - - *plen= *plen - 1; - /* read /n left in the buffer */ - ReadConsole((HANDLE)my_coninpfh, (LPVOID)tmp, tmplen, &tmplen, NULL); - } - } - buffer[*plen]= '\0'; - } - - SetConsoleMode((HANDLE)my_coninpfh, state); - /* unlock the console */ - pthread_auto_mutex_free(my_conio_cs); - - return result; -} - -#endif /* _WIN32 */ -- cgit v1.2.1 From ea0a5cb0a4efbca1fc35885599e07baf10dc3e9e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 19 Nov 2021 14:03:51 +0100 Subject: MDEV-27092 Windows - services that have non-ASCII characters do not work with activeCodePage=UTF8 CreateServiceA, OpenServiceA, and couple of other functions do not work correctly with non-ASCII character, in the special case where application has defined activeCodePage=UTF8. Workaround by redefining affected ANSI functions to own wrapper, which works by converting narrow(ANSI) to wide, then calling wide function. Deprecate original ANSI service functions, via declspec, so that we can catch their use. --- cmake/os/Windows.cmake | 4 +- include/my_global.h | 8 ++ sql/mysql_install_db.cc | 3 +- sql/mysqld.cc | 1 + sql/winmain.cc | 1 + sql/winservice.h | 177 +++++++++++++++++++++++++++++++++++++- win/upgrade_wizard/CMakeLists.txt | 1 + win/upgrade_wizard/upgradeDlg.cpp | 18 ++-- 8 files changed, 201 insertions(+), 12 deletions(-) diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index d9a0dfba46a..da75c73d585 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -60,8 +60,8 @@ ENDIF() ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE) ADD_DEFINITIONS(-D_WIN32_WINNT=0x0A00) -# We do not want the windows.h macros min/max -ADD_DEFINITIONS(-DNOMINMAX) +# We do not want the windows.h , or winsvc.h macros min/max +ADD_DEFINITIONS(-DNOMINMAX -DNOSERVICE) # Speed up build process excluding unused header files ADD_DEFINITIONS(-DWIN32_LEAN_AND_MEAN) diff --git a/include/my_global.h b/include/my_global.h index 224909116dd..ed213e69f85 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -29,6 +29,14 @@ #pragma GCC poison __WIN__ #endif +#if defined(_MSC_VER) +/* + Following functions have bugs, when used with UTF-8 active codepage. + #include will use the non-buggy wrappers +*/ +#pragma deprecated("CreateServiceA", "OpenServiceA", "ChangeServiceConfigA") +#endif + /* InnoDB depends on some MySQL internals which other plugins should not need. This is because of InnoDB's foreign key support, "safe" binlog diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index f712e29b843..026ac3e668e 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -30,6 +30,7 @@ #include struct IUnknown; #include +#include #include @@ -916,7 +917,7 @@ end: auto sc_manager = OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS); if (sc_manager) { - auto sc_handle= OpenServiceA(sc_manager,opt_service, DELETE); + auto sc_handle= OpenService(sc_manager,opt_service, DELETE); if (sc_handle) { DeleteService(sc_handle); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 808e590435a..46c28c862f6 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -127,6 +127,7 @@ #ifdef _WIN32 #include #include +#include /* SERVICE_STOPPED, SERVICE_RUNNING etc */ #endif #include diff --git a/sql/winmain.cc b/sql/winmain.cc index f999767cb27..bdad409065d 100644 --- a/sql/winmain.cc +++ b/sql/winmain.cc @@ -55,6 +55,7 @@ #include #include #include +#include static SERVICE_STATUS svc_status{SERVICE_WIN32_OWN_PROCESS}; static SERVICE_STATUS_HANDLE svc_status_handle; diff --git a/sql/winservice.h b/sql/winservice.h index f9ab3eda332..a14ee73d146 100644 --- a/sql/winservice.h +++ b/sql/winservice.h @@ -17,11 +17,22 @@ /* Extract properties of a windows service binary path */ +#pragma once + #ifdef __cplusplus extern "C" { #endif -#include +#include +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4995) +#endif +#include +#ifdef _MSC_VER +#pragma warning(pop) +#endif + typedef struct mysqld_service_properties_st { char mysqld_exe[MAX_PATH]; @@ -32,9 +43,171 @@ typedef struct mysqld_service_properties_st int version_patch; } mysqld_service_properties; -extern int get_mysql_service_properties(const wchar_t *bin_path, +extern int get_mysql_service_properties(const wchar_t *bin_path, mysqld_service_properties *props); + +#if !defined(UNICODE) +/* + The following wrappers workaround Windows bugs + with CreateService/OpenService with ANSI codepage UTF8. + + Apparently, these function in ANSI mode, for this codepage only + do *not* behave as expected (as-if string parameters were + converted to UTF16 and "wide" function were called) +*/ +#include +static inline wchar_t* awstrdup(const char *str) +{ + if (!str) + return NULL; + size_t len= strlen(str) + 1; + wchar_t *wstr= (wchar_t *) malloc(sizeof(wchar_t)*len); + if (MultiByteToWideChar(GetACP(), 0, str, (int)len, wstr, (int)len) == 0) + { + free(wstr); + return NULL; + } + return wstr; +} + +#define AWSTRDUP(dest, src) \ + dest= awstrdup(src); \ + if (src && !dest) \ + { \ + ok= FALSE; \ + last_error = ERROR_OUTOFMEMORY; \ + goto end; \ + } + +static inline SC_HANDLE my_OpenService(SC_HANDLE hSCManager, LPCSTR lpServiceName, DWORD dwDesiredAccess) +{ + wchar_t *w_ServiceName= NULL; + BOOL ok=TRUE; + DWORD last_error=0; + SC_HANDLE sch=NULL; + + AWSTRDUP(w_ServiceName, lpServiceName); + sch= OpenServiceW(hSCManager, w_ServiceName, dwDesiredAccess); + if (!sch) + { + ok= FALSE; + last_error= GetLastError(); + } + +end: + free(w_ServiceName); + if (!ok) + SetLastError(last_error); + return sch; +} + +static inline SC_HANDLE my_CreateService(SC_HANDLE hSCManager, + LPCSTR lpServiceName, LPCSTR lpDisplayName, + DWORD dwDesiredAccess, DWORD dwServiceType, + DWORD dwStartType, DWORD dwErrorControl, + LPCSTR lpBinaryPathName, LPCSTR lpLoadOrderGroup, + LPDWORD lpdwTagId, LPCSTR lpDependencies, + LPCSTR lpServiceStartName, LPCSTR lpPassword) +{ + wchar_t *w_ServiceName= NULL; + wchar_t *w_DisplayName= NULL; + wchar_t *w_BinaryPathName= NULL; + wchar_t *w_LoadOrderGroup= NULL; + wchar_t *w_Dependencies= NULL; + wchar_t *w_ServiceStartName= NULL; + wchar_t *w_Password= NULL; + SC_HANDLE sch = NULL; + DWORD last_error=0; + BOOL ok= TRUE; + + AWSTRDUP(w_ServiceName,lpServiceName); + AWSTRDUP(w_DisplayName,lpDisplayName); + AWSTRDUP(w_BinaryPathName, lpBinaryPathName); + AWSTRDUP(w_LoadOrderGroup, lpLoadOrderGroup); + AWSTRDUP(w_Dependencies, lpDependencies); + AWSTRDUP(w_ServiceStartName, lpServiceStartName); + AWSTRDUP(w_Password, lpPassword); + + sch= CreateServiceW( + hSCManager, w_ServiceName, w_DisplayName, dwDesiredAccess, dwServiceType, + dwStartType, dwErrorControl, w_BinaryPathName, w_LoadOrderGroup, + lpdwTagId, w_Dependencies, w_ServiceStartName, w_Password); + if(!sch) + { + ok= FALSE; + last_error= GetLastError(); + } + +end: + free(w_ServiceName); + free(w_DisplayName); + free(w_BinaryPathName); + free(w_LoadOrderGroup); + free(w_Dependencies); + free(w_ServiceStartName); + free(w_Password); + + if (!ok) + SetLastError(last_error); + return sch; +} + +static inline BOOL my_ChangeServiceConfig(SC_HANDLE hService, DWORD dwServiceType, + DWORD dwStartType, DWORD dwErrorControl, + LPCSTR lpBinaryPathName, LPCSTR lpLoadOrderGroup, + LPDWORD lpdwTagId, LPCSTR lpDependencies, + LPCSTR lpServiceStartName, LPCSTR lpPassword, + LPCSTR lpDisplayName) +{ + wchar_t *w_DisplayName= NULL; + wchar_t *w_BinaryPathName= NULL; + wchar_t *w_LoadOrderGroup= NULL; + wchar_t *w_Dependencies= NULL; + wchar_t *w_ServiceStartName= NULL; + wchar_t *w_Password= NULL; + SC_HANDLE sch = NULL; + DWORD last_error=0; + BOOL ok= TRUE; + + AWSTRDUP(w_DisplayName, lpDisplayName); + AWSTRDUP(w_BinaryPathName, lpBinaryPathName); + AWSTRDUP(w_LoadOrderGroup, lpLoadOrderGroup); + AWSTRDUP(w_Dependencies, lpDependencies); + AWSTRDUP(w_ServiceStartName, lpServiceStartName); + AWSTRDUP(w_Password, lpPassword); + + ok= ChangeServiceConfigW( + hService, dwServiceType, dwStartType, dwErrorControl, w_BinaryPathName, + w_LoadOrderGroup, lpdwTagId, w_Dependencies, w_ServiceStartName, + w_Password, w_DisplayName); + if (!ok) + { + last_error= GetLastError(); + } + +end: + free(w_DisplayName); + free(w_BinaryPathName); + free(w_LoadOrderGroup); + free(w_Dependencies); + free(w_ServiceStartName); + free(w_Password); + + if (last_error) + SetLastError(last_error); + return ok; +} +#undef AWSTRDUP + +#undef OpenService +#define OpenService my_OpenService +#undef ChangeServiceConfig +#define ChangeServiceConfig my_ChangeServiceConfig +#undef CreateService +#define CreateService my_CreateService +#endif + #ifdef __cplusplus } #endif diff --git a/win/upgrade_wizard/CMakeLists.txt b/win/upgrade_wizard/CMakeLists.txt index 20a06a41215..fd3560e1ee6 100644 --- a/win/upgrade_wizard/CMakeLists.txt +++ b/win/upgrade_wizard/CMakeLists.txt @@ -5,6 +5,7 @@ ENDIF() # We need MFC # /permissive- flag does not play well with MFC, disable it. STRING(REPLACE "/permissive-" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +REMOVE_DEFINITIONS(-DNOSERVICE) # fixes "already defined" warning in an AFX header FIND_PACKAGE(MFC) IF(NOT MFC_FOUND) diff --git a/win/upgrade_wizard/upgradeDlg.cpp b/win/upgrade_wizard/upgradeDlg.cpp index 80f7c18f757..7aae4890b51 100644 --- a/win/upgrade_wizard/upgradeDlg.cpp +++ b/win/upgrade_wizard/upgradeDlg.cpp @@ -141,24 +141,24 @@ void CUpgradeDlg::PopulateServicesList() ErrorExit("OpenSCManager failed"); } - static BYTE buf[64*1024]; + static BYTE buf[2*64*1024]; static BYTE configBuffer[8*1024]; DWORD bufsize= sizeof(buf); DWORD bufneed; DWORD num_services; - BOOL ok= EnumServicesStatusEx(scm, SC_ENUM_PROCESS_INFO, SERVICE_WIN32, + BOOL ok= EnumServicesStatusExW(scm, SC_ENUM_PROCESS_INFO, SERVICE_WIN32, SERVICE_STATE_ALL, buf, bufsize, &bufneed, &num_services, NULL, NULL); if(!ok) ErrorExit("EnumServicesStatusEx failed"); - LPENUM_SERVICE_STATUS_PROCESS info = - (LPENUM_SERVICE_STATUS_PROCESS)buf; + LPENUM_SERVICE_STATUS_PROCESSW info = + (LPENUM_SERVICE_STATUS_PROCESSW)buf; int index=-1; for (ULONG i=0; i < num_services; i++) { - SC_HANDLE service= OpenService(scm, info[i].lpServiceName, + SC_HANDLE service= OpenServiceW(scm, info[i].lpServiceName, SERVICE_QUERY_CONFIG); if (!service) continue; @@ -187,7 +187,11 @@ void CUpgradeDlg::PopulateServicesList() ServiceProperties props; props.myini= service_props.inifile; props.datadir= service_props.datadir; - props.servicename = info[i].lpServiceName; + char service_name_buf[1024]; + WideCharToMultiByte(GetACP(), 0, info[i].lpServiceName, -1, + service_name_buf, sizeof(service_name_buf), + 0, 0); + props.servicename= service_name_buf; if (service_props.version_major) { char ver[64]; @@ -198,7 +202,7 @@ void CUpgradeDlg::PopulateServicesList() else props.version= ""; - index = m_Services.AddString(info[i].lpServiceName); + index = m_Services.AddString(service_name_buf); services.resize(index+1); services[index] = props; } -- cgit v1.2.1 From 4d3ac328482ea2e363cb0be00fd8654d0af5cce0 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 19 Nov 2021 14:14:38 +0100 Subject: MDEV-27093 Do not pass root password in HEX(clear text) from mariadb-install-db.exe to bootstrap Previously, password was passed as hex(clear_text_password). The hex encoding was used to avoid masking apostrophe and backslash etc. However, bootstrap still manages to misinterpert UTF8 password, so that root would not connect later. So the fix is to compute the native password hash inside mysql_install_db already instead, and create user with that hash, rather than letting bootstrap calculate it by using PASSWORD() function. --- sql/CMakeLists.txt | 3 ++- sql/mysql_install_db.cc | 27 +++++++++------------------ 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 972cc0b736c..848495190b2 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -481,10 +481,11 @@ IF(WIN32) MYSQL_ADD_EXECUTABLE(mariadb-install-db mysql_install_db.cc ${CMAKE_CURRENT_BINARY_DIR}/mysql_bootstrap_sql.c + password.c COMPONENT Server ) SET_TARGET_PROPERTIES(mariadb-install-db PROPERTIES COMPILE_FLAGS -DINSTALL_PLUGINDIR=${INSTALL_PLUGINDIR}) - TARGET_LINK_LIBRARIES(mariadb-install-db mysys shlwapi) + TARGET_LINK_LIBRARIES(mariadb-install-db mysys mysys_ssl shlwapi) ADD_LIBRARY(winservice STATIC winservice.c) TARGET_LINK_LIBRARIES(winservice shell32) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index 026ac3e668e..d2fcb5858a8 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -21,6 +21,7 @@ #include "mariadb.h" #include #include +#include #include #include @@ -443,16 +444,14 @@ static int create_myini() } -static const char update_root_passwd_part1[]= +static constexpr const char* update_root_passwd= "UPDATE mysql.global_priv SET priv=json_set(priv," "'$.password_last_changed', UNIX_TIMESTAMP()," "'$.plugin','mysql_native_password'," - "'$.authentication_string',PASSWORD("; -static const char update_root_passwd_part2[]= - ")) where User='root';\n"; -static const char remove_default_user_cmd[]= + "'$.authentication_string','%s') where User='root';\n"; +static constexpr char remove_default_user_cmd[]= "DELETE FROM mysql.user where User='';\n"; -static const char allow_remote_root_access_cmd[]= +static constexpr char allow_remote_root_access_cmd[]= "CREATE TEMPORARY TABLE tmp_user LIKE global_priv;\n" "INSERT INTO tmp_user SELECT * from global_priv where user='root' " " AND host='localhost';\n" @@ -871,18 +870,10 @@ static int create_db_instance(const char *datadir) /* Change root password if requested. */ if (opt_password && opt_password[0]) { - verbose("Setting root password",remove_default_user_cmd); - fputs(update_root_passwd_part1, in); - - /* Use hex encoding for password, to avoid escaping problems.*/ - fputc('0', in); - fputc('x', in); - for(int i= 0; opt_password[i]; i++) - { - fprintf(in,"%02x",opt_password[i]); - } - - fputs(update_root_passwd_part2, in); + verbose("Setting root password"); + char buf[2 * MY_SHA1_HASH_SIZE + 2]; + my_make_scrambled_password(buf, opt_password, strlen(opt_password)); + fprintf(in, update_root_passwd, buf); fflush(in); } -- cgit v1.2.1 From ba9d231b5ab75bc3614e53bdd95026e5fe9dd565 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 22 Nov 2021 12:29:15 +0100 Subject: MDEV-26713 Set activeCodePage=UTF8 for windows programs - Use corresponding entry in the manifest, as described in https://docs.microsoft.com/en-us/windows/apps/design/globalizing/use-utf8-code-page - If if ANSI codepage is UTF8 (i.e for Windows 1903 and later) Use UTF8 as default client charset Set console codepage(s) to UTF8, in case process is using console - Allow some previously disabled MTR tests, that used Unicode for in "exec", for the recent Windows versions --- cmake/win_compatibility.manifest | 5 ++ mysql-test/include/check_utf8_cli.inc | 3 + mysql-test/include/no_utf8_cli.inc | 3 + mysql-test/main/charset_client_win.test | 1 + mysql-test/main/charset_client_win_utf8mb4.result | 2 + mysql-test/main/charset_client_win_utf8mb4.test | 3 + mysql-test/main/grant_not_windows.result | 8 --- mysql-test/main/grant_not_windows.test | 14 ----- mysql-test/main/grant_utf8_cli.result | 8 +++ mysql-test/main/grant_utf8_cli.test | 13 ++++ mysql-test/suite.pm | 22 +++++++ mysys/charset.c | 12 +++- mysys/get_password.c | 2 +- mysys/my_getopt.c | 53 ++++++++++++++-- mysys/my_init.c | 75 +++++++++++++++++++++++ 15 files changed, 194 insertions(+), 30 deletions(-) create mode 100644 mysql-test/include/check_utf8_cli.inc create mode 100644 mysql-test/include/no_utf8_cli.inc create mode 100644 mysql-test/main/charset_client_win_utf8mb4.result create mode 100644 mysql-test/main/charset_client_win_utf8mb4.test delete mode 100644 mysql-test/main/grant_not_windows.result delete mode 100644 mysql-test/main/grant_not_windows.test create mode 100644 mysql-test/main/grant_utf8_cli.result create mode 100644 mysql-test/main/grant_utf8_cli.test diff --git a/cmake/win_compatibility.manifest b/cmake/win_compatibility.manifest index 2e4b27a6dc4..0e7ce667d68 100644 --- a/cmake/win_compatibility.manifest +++ b/cmake/win_compatibility.manifest @@ -19,4 +19,9 @@ + + + UTF-8 + + diff --git a/mysql-test/include/check_utf8_cli.inc b/mysql-test/include/check_utf8_cli.inc new file mode 100644 index 00000000000..a1fac216446 --- /dev/null +++ b/mysql-test/include/check_utf8_cli.inc @@ -0,0 +1,3 @@ +# Check if utf8 can be used on the command line for --exec +# The real check is done in the suite.pm +# diff --git a/mysql-test/include/no_utf8_cli.inc b/mysql-test/include/no_utf8_cli.inc new file mode 100644 index 00000000000..19f9aa6df42 --- /dev/null +++ b/mysql-test/include/no_utf8_cli.inc @@ -0,0 +1,3 @@ +# Check if utf8 can't be used on the command line for --exec +# The real check is done in the suite.pm +# diff --git a/mysql-test/main/charset_client_win.test b/mysql-test/main/charset_client_win.test index b4a21d4f0a4..c3f649cb7d4 100644 --- a/mysql-test/main/charset_client_win.test +++ b/mysql-test/main/charset_client_win.test @@ -1,2 +1,3 @@ --source include/windows.inc +--source include/no_utf8_cli.inc --exec chcp 1257 > NUL && $MYSQL --default-character-set=auto -e "select @@character_set_client" diff --git a/mysql-test/main/charset_client_win_utf8mb4.result b/mysql-test/main/charset_client_win_utf8mb4.result new file mode 100644 index 00000000000..f7b5d376f9a --- /dev/null +++ b/mysql-test/main/charset_client_win_utf8mb4.result @@ -0,0 +1,2 @@ +@@character_set_client +utf8mb4 diff --git a/mysql-test/main/charset_client_win_utf8mb4.test b/mysql-test/main/charset_client_win_utf8mb4.test new file mode 100644 index 00000000000..2baf0d7c050 --- /dev/null +++ b/mysql-test/main/charset_client_win_utf8mb4.test @@ -0,0 +1,3 @@ +--source include/windows.inc +--source include/check_utf8_cli.inc +--exec $MYSQL --default-character-set=auto -e "select @@character_set_client" diff --git a/mysql-test/main/grant_not_windows.result b/mysql-test/main/grant_not_windows.result deleted file mode 100644 index fedfaf984b2..00000000000 --- a/mysql-test/main/grant_not_windows.result +++ /dev/null @@ -1,8 +0,0 @@ -set names utf8; -create user юзер_юзер@localhost; -grant select on test.* to юзер_юзер@localhost; -user() -юзер_юзер@localhost -revoke all on test.* from юзер_юзер@localhost; -drop user юзер_юзер@localhost; -set names default; diff --git a/mysql-test/main/grant_not_windows.test b/mysql-test/main/grant_not_windows.test deleted file mode 100644 index 55b09232edc..00000000000 --- a/mysql-test/main/grant_not_windows.test +++ /dev/null @@ -1,14 +0,0 @@ - # UTF8 parameters to mysql client do not work on Windows ---source include/not_windows.inc ---source include/not_embedded.inc - -# -# Bug#21432 Database/Table name limited to 64 bytes, not chars, problems with multi-byte -# -set names utf8; -create user юзер_юзер@localhost; -grant select on test.* to юзер_юзер@localhost; ---exec $MYSQL --default-character-set=utf8 --user=юзер_юзер -e "select user()" -revoke all on test.* from юзер_юзер@localhost; -drop user юзер_юзер@localhost; -set names default; diff --git a/mysql-test/main/grant_utf8_cli.result b/mysql-test/main/grant_utf8_cli.result new file mode 100644 index 00000000000..fedfaf984b2 --- /dev/null +++ b/mysql-test/main/grant_utf8_cli.result @@ -0,0 +1,8 @@ +set names utf8; +create user юзер_юзер@localhost; +grant select on test.* to юзер_юзер@localhost; +user() +юзер_юзер@localhost +revoke all on test.* from юзер_юзер@localhost; +drop user юзер_юзер@localhost; +set names default; diff --git a/mysql-test/main/grant_utf8_cli.test b/mysql-test/main/grant_utf8_cli.test new file mode 100644 index 00000000000..bc811d5298e --- /dev/null +++ b/mysql-test/main/grant_utf8_cli.test @@ -0,0 +1,13 @@ +--source include/not_embedded.inc +--source include/check_utf8_cli.inc + +# +# Bug#21432 Database/Table name limited to 64 bytes, not chars, problems with multi-byte +# +set names utf8; +create user юзер_юзер@localhost; +grant select on test.* to юзер_юзер@localhost; +--exec $MYSQL --default-character-set=utf8 --user=юзер_юзер -e "select user()" +revoke all on test.* from юзер_юзер@localhost; +drop user юзер_юзер@localhost; +set names default; diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm index 4cc6b410fa1..ad67117a229 100644 --- a/mysql-test/suite.pm +++ b/mysql-test/suite.pm @@ -87,6 +87,28 @@ sub skip_combinations { $skip{'main/ssl_verify_ip.test'} = 'x509v3 support required' unless $openssl_ver ge "1.0.2"; + sub utf8_command_line_ok() { + if (IS_WINDOWS) { + # Can use UTF8 on command line since Windows 10 1903 (10.0.18362) + # or if OS codepage is set to UTF8 + my($os_name, $os_major, $os_minor, $os_build, $os_id) = Win32::GetOSVersion(); + if($os_major lt 10){ + return 0; + } elsif($os_major gt 10 or $os_minor gt 0 or $os_build ge 18362){ + return 1; + } elsif(Win32::GetACP() eq 65001) { + return 1; + } + return 0; + } + return 1; + } + + $skip{'include/check_utf8_cli.inc'} = 'No utf8 command line support' + unless utf8_command_line_ok(); + + $skip{'include/no_utf8_cli.inc'} = 'Not tested with utf8 command line support' + unless !utf8_command_line_ok(); %skip; } diff --git a/mysys/charset.c b/mysys/charset.c index 19cad76fdf4..2a8ac6e1ca5 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -1517,9 +1517,15 @@ const char* my_default_csname() const char* csname = NULL; #ifdef _WIN32 char cpbuf[64]; - int cp = GetConsoleCP(); - if (cp == 0) - cp = GetACP(); + UINT cp; + if (GetACP() == CP_UTF8) + cp= CP_UTF8; + else + { + cp= GetConsoleCP(); + if (cp == 0) + cp= GetACP(); + } snprintf(cpbuf, sizeof(cpbuf), "cp%d", (int)cp); csname = my_os_charset_to_mysql_charset(cpbuf); #elif defined(HAVE_SETLOCALE) && defined(HAVE_NL_LANGINFO) diff --git a/mysys/get_password.c b/mysys/get_password.c index bdd20d0349b..18286fd9e39 100644 --- a/mysys/get_password.c +++ b/mysys/get_password.c @@ -100,7 +100,7 @@ char *get_tty_password(const char *opt_message) /* Allocate output string, and convert UTF16 password to output codepage. */ - cp= GetConsoleCP(); + cp= GetACP() == CP_UTF8 ? CP_UTF8 : GetConsoleCP(); if (!(to_len= WideCharToMultiByte(cp, 0, wbuf, -1, NULL, 0, NULL, NULL))) DBUG_RETURN(NULL); diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 3fe025ba808..6e9c6334620 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -38,7 +38,7 @@ static double getopt_double(char *arg, const struct my_option *optp, int *err); static void init_variables(const struct my_option *, init_func_p); static void init_one_value(const struct my_option *, void *, longlong); static void fini_one_value(const struct my_option *, void *, longlong); -static int setval(const struct my_option *, void *, char *, my_bool); +static int setval(const struct my_option *, void *, char *, my_bool, const char *); static char *check_struct_option(char *cur_arg, char *key_name); /* @@ -133,6 +133,50 @@ double getopt_ulonglong2double(ulonglong v) return u.dbl; } +#ifdef _WIN32 +/** + + On Windows, if program is running in UTF8 mode, but some arguments are not UTF8. + + This will mostly likely be a sign of old "ANSI" my.ini, and it is likely that + something will go wrong, e.g file access error. +*/ +static void validate_value(const char *key, const char *value, + const char *filename) +{ + MY_STRCOPY_STATUS status; + const struct charset_info_st *cs= &my_charset_utf8mb4_bin; + size_t len; + if (GetACP() != CP_UTF8) + return; + if (!(len= strlen(value))) + return; + cs->cset->well_formed_char_length(cs, value, value + len, len, &status); + if (!status.m_well_formed_error_pos) + return; + if (filename && *filename) + { + my_getopt_error_reporter(WARNING_LEVEL, + "%s: invalid (non-UTF8) characters found for option '%s'" + " in file '%s'", + my_progname, key, filename); + } + else + { + /* + Should never happen, non-UTF8 can be read from option's + file only. + */ + DBUG_ASSERT(0); + my_getopt_error_reporter( + WARNING_LEVEL, "%s: invalid (non-UTF8) characters for option %s", + my_progname, key); + } +} +#else +#define validate_value(key, value, filename) (void)filename +#endif + /** Handle command line options. Sort options. @@ -564,7 +608,7 @@ int handle_options(int *argc, char ***argv, const struct my_option *longopts, } } if ((error= setval(optp, optp->value, argument, - set_maximum_value))) + set_maximum_value,filename))) DBUG_RETURN(error); if (get_one_option(optp, argument, filename)) DBUG_RETURN(EXIT_UNSPECIFIED_ERROR); @@ -610,7 +654,7 @@ int handle_options(int *argc, char ***argv, const struct my_option *longopts, continue; } if ((!option_is_autoset) && - ((error= setval(optp, value, argument, set_maximum_value))) && + ((error= setval(optp, value, argument, set_maximum_value,filename))) && !option_is_loose) DBUG_RETURN(error); if (get_one_option(optp, argument, filename)) @@ -711,7 +755,7 @@ static my_bool get_bool_argument(const struct my_option *opts, */ static int setval(const struct my_option *opts, void *value, char *argument, - my_bool set_maximum_value) + my_bool set_maximum_value, const char *option_file) { int err= 0, res= 0; DBUG_ENTER("setval"); @@ -858,6 +902,7 @@ static int setval(const struct my_option *opts, void *value, char *argument, goto ret; }; } + validate_value(opts->name, argument, option_file); DBUG_RETURN(0); ret: diff --git a/mysys/my_init.c b/mysys/my_init.c index d201d45a4ee..2f21bcb735f 100644 --- a/mysys/my_init.c +++ b/mysys/my_init.c @@ -34,6 +34,7 @@ #endif static void my_win_init(void); static my_bool win32_init_tcp_ip(); +static void setup_codepages(); #else #define my_win_init() #endif @@ -67,6 +68,69 @@ static ulong atoi_octal(const char *str) MYSQL_FILE *mysql_stdin= NULL; static MYSQL_FILE instrumented_stdin; +#ifdef _WIN32 +static UINT orig_console_cp, orig_console_output_cp; + +static void reset_console_cp(void) +{ + /* + We try not to call SetConsoleCP unnecessarily, to workaround a bug on + older Windows 10 (1803), which could switch truetype console fonts to + raster, eventhough SetConsoleCP would be a no-op (switch from UTF8 to UTF8). + */ + if (GetConsoleCP() != orig_console_cp) + SetConsoleCP(orig_console_cp); + if (GetConsoleOutputCP() != orig_console_output_cp) + SetConsoleOutputCP(orig_console_output_cp); +} + +/* + The below fixes discrepancies in console output and + command line parameter encoding. command line is in + ANSI codepage, output to console by default is in OEM, but + we like them to be in the same encoding. + + We do this only if current codepage is UTF8, i.e when we + know we're on Windows that can handle UTF8 well. +*/ +static void setup_codepages() +{ + UINT acp; + BOOL is_a_tty= fileno(stdout) >= 0 && isatty(fileno(stdout)); + + if (is_a_tty) + { + /* + Save console codepages, in case we change them, + to restore them on exit. + */ + orig_console_cp= GetConsoleCP(); + orig_console_output_cp= GetConsoleOutputCP(); + if (orig_console_cp && orig_console_output_cp) + atexit(reset_console_cp); + } + + if ((acp= GetACP()) != CP_UTF8) + return; + + /* + Use setlocale to make mbstowcs/mkdir/getcwd behave, see + https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setlocale-wsetlocale + */ + setlocale(LC_ALL, "en_US.UTF8"); + + if (is_a_tty && (orig_console_cp != acp || orig_console_output_cp != acp)) + { + /* + If ANSI codepage is UTF8, we actually want to switch console + to it as well. + */ + SetConsoleCP(acp); + SetConsoleOutputCP(acp); + } +} +#endif + /** Initialize my_sys functions, resources and variables @@ -337,6 +401,17 @@ static void my_win_init(void) _tzset(); + /* + We do not want text translation (LF->CRLF) + when stdout is console/terminal, it is buggy + */ + if (fileno(stdout) >= 0 && isatty(fileno(stdout))) + (void)setmode(fileno(stdout), O_BINARY); + + if (fileno(stderr) >= 0 && isatty(fileno(stderr))) + (void) setmode(fileno(stderr), O_BINARY); + + setup_codepages(); DBUG_VOID_RETURN; } -- cgit v1.2.1 From a4fc41b6b48a1e699343c20685a0e221002eba43 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 22 Nov 2021 12:34:10 +0100 Subject: MDEV-26713 Treat codepage 65001 as utf8mb4, not utf8mb3 Also, fix the "UTF8" option in MSI, which is responsible for character-set-server setting --- mysys/charset.c | 2 +- win/packaging/extra.wxs.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysys/charset.c b/mysys/charset.c index 2a8ac6e1ca5..326fea26f1a 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -1393,7 +1393,7 @@ static const MY_CSET_OS_NAME charsets[] = #ifdef UNCOMMENT_THIS_WHEN_WL_WL_4024_IS_DONE {"cp54936", "gb18030", my_cs_exact}, #endif - {"cp65001", "utf8", my_cs_exact}, + {"cp65001", "utf8mb4", my_cs_exact}, #else /* not Windows */ diff --git a/win/packaging/extra.wxs.in b/win/packaging/extra.wxs.in index 11fe60719ec..dd76cc816f5 100644 --- a/win/packaging/extra.wxs.in +++ b/win/packaging/extra.wxs.in @@ -461,7 +461,7 @@ Section="mysqld" Name="my.ini" Key="character-set-server" - Value="utf8" /> + Value="utf8mb4" /> -- cgit v1.2.1 From a296c52627fc46b400d4b9e79dad1e6d8690b8cc Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 22 Nov 2021 13:22:05 +0100 Subject: MDEV-26713 UTF8 support on Windows, mysql_upgrade_service preparation - Tolerate situation, when datadir for service seems invalid/non-existing prior to upgrade. It could be that my.ini contains legacy ANSI characters for the data directory. Those can't be read correctly by mysql_upgrade_service, which uses a different ANSI codepage(UTF8) . - schedule upgrade_config_file at later stage, because once we convert it to UTF-8 (followup patch), this will render config file uselss with the older version of mariadbd.exe - Refactor upgrade_conf_file.cc, prepare for UTF-8 conversion. --- sql/mysql_upgrade_service.cc | 31 ++++++----- sql/upgrade_conf_file.cc | 124 ++++++++++++++++++++++++++++++------------- sql/winservice.c | 25 +++++---- 3 files changed, 123 insertions(+), 57 deletions(-) diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc index 19dbf93c7ce..ce16e10eeff 100644 --- a/sql/mysql_upgrade_service.cc +++ b/sql/mysql_upgrade_service.cc @@ -374,13 +374,17 @@ static void change_service_config() Write datadir to my.ini, after converting backslashes to unix style slashes. */ - strcpy_s(buf, MAX_PATH, service_properties.datadir); - for(i= 0; buf[i]; i++) + if (service_properties.datadir[0]) { - if (buf[i] == '\\') - buf[i]= '/'; + strcpy_s(buf, MAX_PATH, service_properties.datadir); + for (i= 0; buf[i]; i++) + { + if (buf[i] == '\\') + buf[i]= '/'; + } + WritePrivateProfileString("mysqld", "datadir", buf, + service_properties.inifile); } - WritePrivateProfileString("mysqld", "datadir",buf, service_properties.inifile); /* Remove basedir from defaults file, otherwise the service wont come up in @@ -465,13 +469,8 @@ int main(int argc, char **argv) } } - old_mysqld_exe_exists = (GetFileAttributes(service_properties.mysqld_exe) != INVALID_FILE_ATTRIBUTES); - log("Phase %d/%d: Fixing server config file%s", ++phase, max_phases, my_ini_exists ? "" : "(skipped)"); - - snprintf(my_ini_bck, sizeof(my_ini_bck), "%s.BCK", service_properties.inifile); - CopyFile(service_properties.inifile, my_ini_bck, FALSE); - upgrade_config_file(service_properties.inifile); - + old_mysqld_exe_exists= (GetFileAttributes(service_properties.mysqld_exe) != + INVALID_FILE_ATTRIBUTES); bool do_start_stop_server = old_mysqld_exe_exists && initial_service_state != SERVICE_RUNNING; log("Phase %d/%d: Start and stop server in the old version, to avoid crash recovery %s", ++phase, max_phases, @@ -526,6 +525,14 @@ int main(int argc, char **argv) start_duration_ms += 500; } } + + log("Phase %d/%d: Fixing server config file%s", ++phase, max_phases, + my_ini_exists ? "" : "(skipped)"); + snprintf(my_ini_bck, sizeof(my_ini_bck), "%s.BCK", + service_properties.inifile); + CopyFile(service_properties.inifile, my_ini_bck, FALSE); + upgrade_config_file(service_properties.inifile); + /* Start mysqld.exe as non-service skipping privileges (so we do not care about the password). But disable networking and enable pipe diff --git a/sql/upgrade_conf_file.cc b/sql/upgrade_conf_file.cc index 543df7b9bdf..66d78595164 100644 --- a/sql/upgrade_conf_file.cc +++ b/sql/upgrade_conf_file.cc @@ -158,51 +158,103 @@ static int cmp_strings(const void* a, const void *b) return strcmp((const char *)a, *(const char **)b); } -/** - Convert file from a previous version, by removing -*/ -int upgrade_config_file(const char *myini_path) + +#define MY_INI_SECTION_SIZE 32 * 1024 + 3 + + +int fix_section(const char *myini_path, const char *section_name, + bool is_server) { -#define MY_INI_SECTION_SIZE 32*1024 +3 + if (!is_server) + return 0; + static char section_data[MY_INI_SECTION_SIZE]; - for (const char *section_name : { "mysqld","server","mariadb" }) + DWORD size= GetPrivateProfileSection(section_name, section_data, + MY_INI_SECTION_SIZE, myini_path); + if (size == MY_INI_SECTION_SIZE - 2) { - DWORD size = GetPrivateProfileSection(section_name, section_data, MY_INI_SECTION_SIZE, myini_path); - if (size == MY_INI_SECTION_SIZE - 2) - { - return -1; - } + return -1; + } - for (char *keyval = section_data; *keyval; keyval += strlen(keyval) + 1) + for (char *keyval= section_data; *keyval; keyval += strlen(keyval)+1) + { + char varname[256]; + char *value; + char *key_end= strchr(keyval, '='); + if (!key_end) + key_end= keyval + strlen(keyval); + + if (key_end - keyval > sizeof(varname)) + continue; + + value= key_end + 1; + + // Check if variable should be removed from config. + // First, copy and normalize (convert dash to underscore) to variable + // names + for (char *p= keyval, *q= varname;; p++, q++) { - char varname[256]; - char *key_end = strchr(keyval, '='); - if (!key_end) - key_end = keyval+ strlen(keyval); - - if (key_end - keyval > sizeof(varname)) - continue; - // copy and normalize (convert dash to underscore) to variable names - for (char *p = keyval, *q = varname;; p++,q++) + if (p == key_end) { - if (p == key_end) - { - *q = 0; - break; - } - *q = (*p == '-') ? '_' : *p; + *q= 0; + break; } - const char *v = (const char *)bsearch(varname, removed_variables, sizeof(removed_variables) / sizeof(removed_variables[0]), - sizeof(char *), cmp_strings); + *q= (*p == '-') ? '_' : *p; + } + const char *v= (const char *) bsearch(varname, removed_variables, sizeof(removed_variables) / sizeof(removed_variables[0]), + sizeof(char *), cmp_strings); - if (v) - { - fprintf(stdout, "Removing variable '%s' from config file\n", varname); - // delete variable - *key_end = 0; - WritePrivateProfileString(section_name, keyval, 0, myini_path); - } + if (v) + { + fprintf(stdout, "Removing variable '%s' from config file\n", varname); + // delete variable + *key_end= 0; + WritePrivateProfileString(section_name, keyval, 0, myini_path); } } return 0; } + +static bool is_mariadb_section(const char *name, bool *is_server) +{ + if (strncmp(name, "mysql", 5) + && strncmp(name, "mariadb", 7) + && strcmp(name, "client") + && strcmp(name, "client-server") + && strcmp(name, "server")) + { + return false; + } + + for (const char *section_name : {"mysqld", "server", "mariadb"}) + if (*is_server= !strcmp(section_name, name)) + break; + + return *is_server; +} + + +/** + Convert file from a previous version, by removing obsolete variables + Also, fix values to be UTF8, if MariaDB is running in utf8 mode +*/ +int upgrade_config_file(const char *myini_path) +{ + static char all_sections[MY_INI_SECTION_SIZE]; + int sz= GetPrivateProfileSectionNamesA(all_sections, MY_INI_SECTION_SIZE, + myini_path); + if (!sz) + return 0; + if (sz > MY_INI_SECTION_SIZE - 2) + { + fprintf(stderr, "Too many sections in config file\n"); + return -1; + } + for (char *section= all_sections; *section; section+= strlen(section) + 1) + { + bool is_server_section; + if (is_mariadb_section(section, &is_server_section)) + fix_section(myini_path, section, is_server_section); + } + return 0; +} diff --git a/sql/winservice.c b/sql/winservice.c index a11087e5cd5..d4e3bb0944d 100644 --- a/sql/winservice.c +++ b/sql/winservice.c @@ -134,6 +134,20 @@ static void get_datadir_from_ini(const char *ini, char *service_name, char *data } +static int fix_and_check_datadir(mysqld_service_properties *props) +{ + normalize_path(props->datadir, MAX_PATH); + /* Check if datadir really exists */ + if (GetFileAttributes(props->datadir) != INVALID_FILE_ATTRIBUTES) + return 0; + /* + It is possible, that datadir contains some unconvertable character. + We just pretend not to know what's the data directory + */ + props->datadir[0]= 0; + return 0; +} + /* Retrieve some properties from windows mysqld service binary path. We're interested in ini file location and datadir, and also in version of @@ -284,16 +298,9 @@ int get_mysql_service_properties(const wchar_t *bin_path, } } - if (props->datadir[0]) - { - normalize_path(props->datadir, MAX_PATH); - /* Check if datadir really exists */ - if (GetFileAttributes(props->datadir) == INVALID_FILE_ATTRIBUTES) - goto end; - } - else + if (props->datadir[0] == 0 || fix_and_check_datadir(props)) { - /* There is no datadir in ini file, bail out.*/ + /* There is no datadir in ini file, or non-existing dir, bail out.*/ goto end; } -- cgit v1.2.1 From 6b4b625da35f228066813d30366602bf5d97d3e6 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 11 Dec 2021 04:25:22 +0100 Subject: MDEV-26713 UTF8 support on Windows, convert my.ini from ANSI to UTF8 Handle upgrade - on Windows that is capable of UTF8 codepage, convert entries in my.ini from ANSI to UTF8, during upgrade Reason is that for the server, paths such as datadir, innodb directories, location of SSL certificates must now be utf8. For the client programs , user name, database etc should be in UTF8, too, as UTF-8 is now the default charset. --- sql/upgrade_conf_file.cc | 60 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/sql/upgrade_conf_file.cc b/sql/upgrade_conf_file.cc index 66d78595164..9a19dd8b8a2 100644 --- a/sql/upgrade_conf_file.cc +++ b/sql/upgrade_conf_file.cc @@ -25,6 +25,8 @@ Note : the list below only includes the default-compiled server and none of the loadable plugins. */ +#include +#include #include #include #include @@ -161,11 +163,52 @@ static int cmp_strings(const void* a, const void *b) #define MY_INI_SECTION_SIZE 32 * 1024 + 3 +static bool is_utf8_str(const char *s) +{ + MY_STRCOPY_STATUS status; + const struct charset_info_st *cs= &my_charset_utf8mb4_bin; + size_t len= strlen(s); + if (!len) + return true; + cs->cset->well_formed_char_length(cs, s, s + len, len, &status); + return status.m_well_formed_error_pos == nullptr; +} + + +static UINT get_system_acp() +{ + static DWORD system_acp; + if (system_acp) + return system_acp; + + char str_cp[10]; + int cch= GetLocaleInfo(GetSystemDefaultLCID(), LOCALE_IDEFAULTANSICODEPAGE, + str_cp, sizeof(str_cp)); + + system_acp= cch > 0 ? atoi(str_cp) : 1252; + + return system_acp; +} + + +static char *ansi_to_utf8(const char *s) +{ +#define MAX_STR_LEN MY_INI_SECTION_SIZE + static wchar_t utf16_buf[MAX_STR_LEN]; + static char utf8_buf[MAX_STR_LEN]; + if (MultiByteToWideChar(get_system_acp(), 0, s, -1, utf16_buf, MAX_STR_LEN)) + { + if (WideCharToMultiByte(CP_UTF8, 0, utf16_buf, -1, utf8_buf, MAX_STR_LEN, + 0, 0)) + return utf8_buf; + } + return 0; +} int fix_section(const char *myini_path, const char *section_name, bool is_server) { - if (!is_server) + if (!is_server && GetACP() != CP_UTF8) return 0; static char section_data[MY_INI_SECTION_SIZE]; @@ -188,6 +231,21 @@ int fix_section(const char *myini_path, const char *section_name, continue; value= key_end + 1; + if (GetACP() == CP_UTF8 && !is_utf8_str(value)) + { + /*Convert a value, if it is not already UTF-8*/ + char *new_val= ansi_to_utf8(value); + if (new_val) + { + *key_end= 0; + fprintf(stdout, "Fixing variable '%s' charset, value=%s\n", keyval, + new_val); + WritePrivateProfileString(section_name, keyval, new_val, myini_path); + *key_end= '='; + } + } + if (!is_server) + continue; // Check if variable should be removed from config. // First, copy and normalize (convert dash to underscore) to variable -- cgit v1.2.1 From 825f9c7b50d488ce525cc9c55eda449aa7a59b47 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 23 Nov 2021 12:28:28 +0100 Subject: MDEV-27227 mysqltest, Windows - support background execution in 'exec' command If last character in command is ampersand, do not use my_popen/my_pclose in "exec", instead do CreateProcess() without a window. --- client/mysqltest.cc | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index d1b9302370a..84b04435f1b 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -3251,6 +3251,47 @@ static int replace(DYNAMIC_STRING *ds_str, return 0; } +#ifdef _WIN32 +/** + Check if background execution of command was requested. + Like in Unix shell, we assume background execution of the last + character in command is a ampersand (we do not tokenize though) +*/ +static bool is_background_command(const DYNAMIC_STRING *ds) +{ + for (size_t i= ds->length - 1; i > 1; i--) + { + char c= ds->str[i]; + if (!isspace(c)) + return (c == '&'); + } + return false; +} + +/** + Execute OS command in background. We assume that the last character + is ampersand, i.e is_background_command() returned +*/ +#include +static int execute_in_background(char *cmd) +{ + STARTUPINFO s{}; + PROCESS_INFORMATION pi{}; + char *end= strrchr(cmd, '&'); + DBUG_ASSERT(end); + *end =0; + std::string scmd("cmd /c "); + scmd.append(cmd); + BOOL ok= + CreateProcess(0, (char *)scmd.c_str(), 0, 0, 0, CREATE_NO_WINDOW, 0, 0, &s, &pi); + *end= '&'; + if (!ok) + return (int) GetLastError(); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + return 0; +} +#endif /* Execute given command. @@ -3325,6 +3366,14 @@ void do_exec(struct st_command *command) DBUG_PRINT("info", ("Executing '%s' as '%s'", command->first_argument, ds_cmd.str)); +#ifdef _WIN32 + if (is_background_command(&ds_cmd)) + { + error= execute_in_background(ds_cmd.str); + goto end; + } +#endif + if (!(res_file= my_popen(ds_cmd.str, "r"))) { dynstr_free(&ds_cmd); @@ -3351,7 +3400,9 @@ void do_exec(struct st_command *command) dynstr_append_sorted(&ds_res, &ds_sorted, 0); dynstr_free(&ds_sorted); } - +#ifdef _WIN32 +end: +#endif if (error) { uint status= WEXITSTATUS(error); -- cgit v1.2.1 From 57d52657a27d702e86e99097d59195a9ad4cd968 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 23 Nov 2021 13:05:25 +0100 Subject: MDEV-26713 UTF8 support on Windows , add mysql_install_db tests Add mysql_install_db test with some i18n, for data dir and root password --- mysql-test/main/mysql_install_db_win_utf8.result | 14 ++++++++++ mysql-test/main/mysql_install_db_win_utf8.test | 35 ++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 mysql-test/main/mysql_install_db_win_utf8.result create mode 100644 mysql-test/main/mysql_install_db_win_utf8.test diff --git a/mysql-test/main/mysql_install_db_win_utf8.result b/mysql-test/main/mysql_install_db_win_utf8.result new file mode 100644 index 00000000000..744c982b291 --- /dev/null +++ b/mysql-test/main/mysql_install_db_win_utf8.result @@ -0,0 +1,14 @@ +Running bootstrap +Creating my.ini file +Removing default user +Allowing remote access for user root +Setting root password +Creation of the database was successful +# Kill the server +connect con1,localhost,root,パスワード,mysql; +SELECT @@datadir; +@@datadir +DATADIR/ +# Kill the server +connection default; +# restart diff --git a/mysql-test/main/mysql_install_db_win_utf8.test b/mysql-test/main/mysql_install_db_win_utf8.test new file mode 100644 index 00000000000..fc67b66d3ca --- /dev/null +++ b/mysql-test/main/mysql_install_db_win_utf8.test @@ -0,0 +1,35 @@ +--source include/windows.inc +--source include/check_utf8_cli.inc + +# Create database in tmp directory using mysql_install_db.exe, +# and start server from this directory. +let $ddir= $MYSQLTEST_VARDIR/tmp/датадир; +--error 0,1 +rmdir $ddir; +exec $MYSQL_INSTALL_DB_EXE --datadir=$ddir --password=パスワード -R; +--source include/kill_mysqld.inc + +# Note "restart" via MTR does not work, if server's command line has +# non-ASCII characters used (or, characters outside of ANSI codepage). +# This is a perl limitation, which is worked around in this test - +# the server started in background, via exec $MYSQLD + +--replace_result $MYSQLD MYSQLD $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +exec $MYSQLD --defaults-file=$MYSQLTEST_VARDIR/my.cnf --defaults-group-suffix=.1 --datadir=$ddir --loose-innodb > NUL 2>&1 &; +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect + +connect (con1,localhost,root,パスワード,mysql); + +# Smoke test - check that we're actually using datadir +# we've created (i.e restart_parameters worked) +--replace_result $ddir DATADIR +SELECT @@datadir; +# restart in the original datadir again +--source include/kill_mysqld.inc +rmdir $ddir; + +connection default; +--source include/start_mysqld.inc + -- cgit v1.2.1 From 74f2e6c85e892b6b9e8d65d14db3246d34cc6fc7 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 24 Nov 2021 10:15:11 +0100 Subject: MDEV-26713 Add test for mysql_install_db creating service, with i18 --- mysql-test/include/check_windows_admin.inc | 3 ++ mysql-test/main/winservice.inc | 75 ++++++++++++++++++++++++++++++ mysql-test/main/winservice_basic.result | 12 +++++ mysql-test/main/winservice_basic.test | 5 ++ mysql-test/main/winservice_i18n.result | 12 +++++ mysql-test/main/winservice_i18n.test | 7 +++ mysql-test/suite.pm | 3 ++ 7 files changed, 117 insertions(+) create mode 100644 mysql-test/include/check_windows_admin.inc create mode 100644 mysql-test/main/winservice.inc create mode 100644 mysql-test/main/winservice_basic.result create mode 100644 mysql-test/main/winservice_basic.test create mode 100644 mysql-test/main/winservice_i18n.result create mode 100644 mysql-test/main/winservice_i18n.test diff --git a/mysql-test/include/check_windows_admin.inc b/mysql-test/include/check_windows_admin.inc new file mode 100644 index 00000000000..44a8b71a19f --- /dev/null +++ b/mysql-test/include/check_windows_admin.inc @@ -0,0 +1,3 @@ +# Check if current user is Windows admin +# Used for testing services with mysql_install_db.exe +# Actual value is set by suite.pm diff --git a/mysql-test/main/winservice.inc b/mysql-test/main/winservice.inc new file mode 100644 index 00000000000..42aab645cc8 --- /dev/null +++ b/mysql-test/main/winservice.inc @@ -0,0 +1,75 @@ +source include/check_windows_admin.inc; + +# The test uses return code from sc.exe utility, which are as follows +let $ERROR_SERVICE_DOES_NOT_EXIST= 1060; +let $ERROR_SERVICE_CANNOT_ACCEPT_CTRL=1061;# intermediate, during start or stop +let $ERROR_SERVICE_NOT_ACTIVE=1062;# service stopped + +let $sc_exe= C:\Windows\System32\sc.exe; +let $ddir= $MYSQLTEST_VARDIR/tmp/$datadir_name; +let $service_name=$service_name_prefix$MASTER_MYPORT; + + +error 0,1; +rmdir $ddir; + +--disable_result_log +error 0,$ERROR_SERVICE_DOES_NOT_EXIST; +exec $sc_exe delete $service_name; +--enable_result_log + +source include/kill_mysqld.inc; +echo # run mysql_install_db with --service parameter; +--disable_result_log +exec $MYSQL_INSTALL_DB_EXE --datadir=$ddir --port=$MASTER_MYPORT --password=$password --service=$service_name -R; +--enable_result_log + +echo # Start service; +--disable_result_log +exec $sc_exe start $service_name; +--enable_result_log + +enable_reconnect; +source include/wait_until_connected_again.inc; +disable_reconnect; + +echo # Connect with root user password=$password; +connect (con1,localhost,root,$password,mysql); + +# Smoke test - check that we're actually using datadir +# we've created (i.e restart_parameters worked) +replace_result $ddir DATADIR; +select @@datadir; + +echo # Stop service and wait until it is down; + +# stop service +--disable_result_log +exec $sc_exe stop $service_name; +# Wait until stopped +let $sys_errno=0; +while($sys_errno != $ERROR_SERVICE_NOT_ACTIVE) +{ + --error 0,$ERROR_SERVICE_CANNOT_ACCEPT_CTRL,$ERROR_SERVICE_NOT_ACTIVE + exec $sc_exe stop $service_name; + if($sys_errno != $ERROR_SERVICE_NOT_ACTIVE) + { + --real_sleep 0.1 + } +} +--enable_result_log + +echo # Delete service; +let $sys_errno=0; +--disable_result_log +exec $sc_exe delete $service_name; +--enable_result_log + +# Cleanup +source include/wait_until_disconnected.inc; +rmdir $ddir; + +#restart original server +connection default; +source include/start_mysqld.inc; + diff --git a/mysql-test/main/winservice_basic.result b/mysql-test/main/winservice_basic.result new file mode 100644 index 00000000000..a4de0080207 --- /dev/null +++ b/mysql-test/main/winservice_basic.result @@ -0,0 +1,12 @@ +# Kill the server +# run mysql_install_db with --service parameter +# Start service +# Connect with root user password=password +connect con1,localhost,root,$password,mysql; +select @@datadir; +@@datadir +DATADIR/ +# Stop service and wait until it is down +# Delete service +connection default; +# restart diff --git a/mysql-test/main/winservice_basic.test b/mysql-test/main/winservice_basic.test new file mode 100644 index 00000000000..33d072b4f70 --- /dev/null +++ b/mysql-test/main/winservice_basic.test @@ -0,0 +1,5 @@ +source include/windows.inc; +let $datadir_name=data; +let $service_name_prefix=mariadb; +let $password=password; +source winservice.inc; diff --git a/mysql-test/main/winservice_i18n.result b/mysql-test/main/winservice_i18n.result new file mode 100644 index 00000000000..873f0828b1f --- /dev/null +++ b/mysql-test/main/winservice_i18n.result @@ -0,0 +1,12 @@ +# Kill the server +# run mysql_install_db with --service parameter +# Start service +# Connect with root user password=パスワード +connect con1,localhost,root,$password,mysql; +select @@datadir; +@@datadir +DATADIR/ +# Stop service and wait until it is down +# Delete service +connection default; +# restart diff --git a/mysql-test/main/winservice_i18n.test b/mysql-test/main/winservice_i18n.test new file mode 100644 index 00000000000..c63b38fd2b8 --- /dev/null +++ b/mysql-test/main/winservice_i18n.test @@ -0,0 +1,7 @@ +source include/windows.inc; +source include/check_utf8_cli.inc; + +let $datadir_name=датадир; +let $service_name_prefix=mariadb_sörvis; +let $password=パスワード; +source winservice.inc; diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm index ad67117a229..f30cc5ec431 100644 --- a/mysql-test/suite.pm +++ b/mysql-test/suite.pm @@ -109,6 +109,9 @@ sub skip_combinations { $skip{'include/no_utf8_cli.inc'} = 'Not tested with utf8 command line support' unless !utf8_command_line_ok(); + + $skip{'include/check_windows_admin.inc'} = 'Requires admin privileges' + unless IS_WINDOWS and Win32::IsAdminUser(); %skip; } -- cgit v1.2.1 From 9ea83f7fbd6fe3f76ba0304851cfe96cd4cc1071 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 29 Nov 2021 19:47:36 +0100 Subject: MDEV-26713 set console codepage to what user set in --default-character-set If someone on whatever reasons uses --default-character-set=cp850, this will avoid incorrect display, and inserting incorrect data. Adjusting console codepage sometimes also needs to happen with --default-charset=auto, on older Windows. This is because autodetection is not always exact. For example, console codepage on US editions of Windows is 437. Client autodetects it as cp850, a rather loose approximation, given 46 code point differences. We change the console codepage to cp850, so that there is no discrepancy. That fix is currently Windows-only, and serves people who used combination of chcp to achieve WYSIWYG effect (although, this would mostly likely used with utf8 in the past) Now, --default-character-set would be a replacement for that. Fix fs_character_set() detection of current codepage. --- client/mysql.cc | 17 ++++++++ client/mysqladmin.cc | 1 + client/mysqlcheck.c | 1 + client/mysqlimport.c | 1 + client/mysqlshow.c | 1 + include/my_sys.h | 3 ++ mysys/charset.c | 120 +++++++++++++++++++++++++++++++++++++++++---------- 7 files changed, 121 insertions(+), 23 deletions(-) diff --git a/client/mysql.cc b/client/mysql.cc index 01a511b35b9..d15a5ebdb40 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -3304,6 +3304,21 @@ com_clear(String *buffer,char *line __attribute__((unused))) return 0; } +static void adjust_console_codepage(const char *name __attribute__((unused))) +{ +#ifdef _WIN32 + if (my_set_console_cp(name) < 0) + { + char buf[128]; + snprintf(buf, sizeof(buf), + "WARNING: Could not determine Windows codepage for charset '%s'," + "continue using codepage %u", name, GetConsoleOutputCP()); + put_info(buf, INFO_INFO); + } +#endif +} + + /* ARGSUSED */ static int com_charset(String *buffer __attribute__((unused)), char *line) @@ -3325,6 +3340,7 @@ com_charset(String *buffer __attribute__((unused)), char *line) mysql_set_character_set(&mysql, charset_info->cs_name.str); default_charset= (char *)charset_info->cs_name.str; put_info("Charset changed", INFO_INFO); + adjust_console_codepage(charset_info->cs_name.str); } else put_info("Charset is not found", INFO_INFO); return 0; @@ -4873,6 +4889,7 @@ sql_real_connect(char *host,char *database,char *user,char *password, put_info(buff, INFO_ERROR); return 1; } + adjust_console_codepage(charset_info->cs_name.str); connected=1; #ifndef EMBEDDED_LIBRARY mysql_options(&mysql, MYSQL_OPT_RECONNECT, &debug_info_flag); diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc index 6fa5d6c73d0..a7159d2bb6a 100644 --- a/client/mysqladmin.cc +++ b/client/mysqladmin.cc @@ -438,6 +438,7 @@ int main(int argc,char *argv[]) mysql_options(&mysql,MYSQL_OPT_PROTOCOL,(char*)&opt_protocol); if (!strcmp(default_charset,MYSQL_AUTODETECT_CHARSET_NAME)) default_charset= (char *)my_default_csname(); + my_set_console_cp(default_charset); mysql_options(&mysql, MYSQL_SET_CHARSET_NAME, default_charset); error_flags= (myf)(opt_nobeep ? 0 : ME_BELL); diff --git a/client/mysqlcheck.c b/client/mysqlcheck.c index 480308aa015..eb063765a37 100644 --- a/client/mysqlcheck.c +++ b/client/mysqlcheck.c @@ -503,6 +503,7 @@ static int get_options(int *argc, char ***argv) printf("Unsupported character set: %s\n", default_charset); DBUG_RETURN(1); } + my_set_console_cp(default_charset); if (*argc > 0 && opt_alldbs) { printf("You should give only options, no arguments at all, with option\n"); diff --git a/client/mysqlimport.c b/client/mysqlimport.c index 8723641c74b..48f11667cd1 100644 --- a/client/mysqlimport.c +++ b/client/mysqlimport.c @@ -525,6 +525,7 @@ static MYSQL *db_connect(char *host, char *database, mysql_options(mysql, MYSQL_DEFAULT_AUTH, opt_default_auth); if (!strcmp(default_charset,MYSQL_AUTODETECT_CHARSET_NAME)) default_charset= (char *)my_default_csname(); + my_set_console_cp(default_charset); mysql_options(mysql, MYSQL_SET_CHARSET_NAME, my_default_csname()); mysql_options(mysql, MYSQL_OPT_CONNECT_ATTR_RESET, 0); mysql_options4(mysql, MYSQL_OPT_CONNECT_ATTR_ADD, diff --git a/client/mysqlshow.c b/client/mysqlshow.c index 9b31d87225c..cbac1817c3c 100644 --- a/client/mysqlshow.c +++ b/client/mysqlshow.c @@ -147,6 +147,7 @@ int main(int argc, char **argv) if (!strcmp(default_charset,MYSQL_AUTODETECT_CHARSET_NAME)) default_charset= (char *)my_default_csname(); + my_set_console_cp(default_charset); mysql_options(&mysql, MYSQL_SET_CHARSET_NAME, default_charset); if (opt_plugin_dir && *opt_plugin_dir) diff --git a/include/my_sys.h b/include/my_sys.h index 5a4608155e4..7186cd03cb1 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -1086,6 +1086,9 @@ extern char *get_tty_password(const char *opt_message); #define BACKSLASH_MBTAIL /* File system character set */ extern CHARSET_INFO *fs_character_set(void); +extern int my_set_console_cp(const char *name); +#else +#define my_set_console_cp(A) do {} while (0) #endif extern const char *my_default_csname(void); extern size_t escape_quotes_for_mysql(CHARSET_INFO *charset_info, diff --git a/mysys/charset.c b/mysys/charset.c index 326fea26f1a..da6180dccba 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -1209,30 +1209,17 @@ size_t escape_string_for_mysql(CHARSET_INFO *charset_info, #ifdef BACKSLASH_MBTAIL -static CHARSET_INFO *fs_cset_cache= NULL; - CHARSET_INFO *fs_character_set() { - if (!fs_cset_cache) - { - char buf[10]= "cp"; - GetLocaleInfo(LOCALE_SYSTEM_DEFAULT, LOCALE_IDEFAULTANSICODEPAGE, - buf+2, sizeof(buf)-3); - /* - We cannot call get_charset_by_name here - because fs_character_set() is executed before - LOCK_THD_charset mutex initialization, which - is used inside get_charset_by_name. - As we're now interested in cp932 only, - let's just detect it using strcmp(). - */ - fs_cset_cache= - #ifdef HAVE_CHARSET_cp932 - !strcmp(buf, "cp932") ? &my_charset_cp932_japanese_ci : - #endif - &my_charset_bin; - } - return fs_cset_cache; + static CHARSET_INFO *fs_cset_cache; + if (fs_cset_cache) + return fs_cset_cache; +#ifdef HAVE_CHARSET_cp932 + else if (GetACP() == 932) + return fs_cset_cache= &my_charset_cp932_japanese_ci; +#endif + else + return fs_cset_cache= &my_charset_bin; } #endif @@ -1394,7 +1381,7 @@ static const MY_CSET_OS_NAME charsets[] = {"cp54936", "gb18030", my_cs_exact}, #endif {"cp65001", "utf8mb4", my_cs_exact}, - + {"cp65001", "utf8mb3", my_cs_approx}, #else /* not Windows */ {"646", "latin1", my_cs_approx}, /* Default on Solaris */ @@ -1534,3 +1521,90 @@ const char* my_default_csname() #endif return csname ? csname : MYSQL_DEFAULT_CHARSET_NAME; } + + +#ifdef _WIN32 +/** + Extract codepage number from "cpNNNN" string, + and check that this codepage is supported. + + @return 0 - invalid codepage(or unsupported) + > 0 - valid codepage number. +*/ +static UINT get_codepage(const char *s) +{ + UINT cp; + if (s[0] != 'c' || s[1] != 'p') + { + DBUG_ASSERT(0); + return 0; + } + cp= strtoul(s + 2, NULL, 10); + if (!IsValidCodePage(cp)) + { + /* + Can happen also with documented CP, i.e 51936 + Perhaps differs from one machine to another. + */ + return 0; + } + return cp; +} + +static UINT mysql_charset_to_codepage(const char *my_cs_name) +{ + const MY_CSET_OS_NAME *csp; + UINT cp=0,tmp; + for (csp= charsets; csp->os_name; csp++) + { + if (!strcasecmp(csp->my_name, my_cs_name)) + { + switch (csp->param) + { + case my_cs_exact: + tmp= get_codepage(csp->os_name); + if (tmp) + return tmp; + break; + case my_cs_approx: + /* + don't return just yet, perhaps there is a better + (exact) match later. + */ + if (!cp) + cp= get_codepage(csp->os_name); + continue; + + default: + return 0; + } + } + } + return cp; +} + +/** Set console codepage for MariaDB's charset name */ +int my_set_console_cp(const char *csname) +{ + UINT cp; + if (fileno(stdout) < 0 || !isatty(fileno(stdout))) + return 0; + cp= mysql_charset_to_codepage(csname); + if (!cp) + { + /* No compatible os charset.*/ + return -1; + } + + if (GetConsoleOutputCP() != cp && !SetConsoleOutputCP(cp)) + { + return -1; + } + + if (GetConsoleCP() != cp && !SetConsoleCP(cp)) + { + return -1; + } + return 0; +} +#endif -- cgit v1.2.1 From 71966c7306d6ae24bc052d5433e6d97cc81a436a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 26 Nov 2021 18:41:35 +0100 Subject: MDEV-26713 allow users with non-UTF8 passwords to login after upgrade. Translate username, password and database from UTF8 into desired charset, if non-auto default-character-set was used, on Windows10 1903 This change is implemented only in the command line client, and mainly to allow users with non-UTF8 passwords to login. The user is supposed to use the same charset that was used during setting password (usually, console CP if used in CLI) Add a test to document the behavior. --- client/mysql.cc | 44 +++++++++++++++++++++++ mysql-test/main/charset_client_win_utf8mb4.result | 4 +++ mysql-test/main/charset_client_win_utf8mb4.test | 19 ++++++++++ 3 files changed, 67 insertions(+) diff --git a/client/mysql.cc b/client/mysql.cc index d15a5ebdb40..49e6d29e964 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -1442,6 +1442,46 @@ sig_handler mysql_end(int sig) exit(status.exit_status); } +#ifdef _WIN32 +#define CNV_BUFSIZE 1024 + +/** + Convert user,database,and password to requested charset. + + This is done in the single case when user connects with non-UTF8 + default-character-set, on UTF8 capable Windows. + + User, password, and database are UTF8 encoded, prior to the function, + this needs to be fixed, in case they contain non-ASCIIs. + + Mostly a workaround, to allow existng users with non-ASCII password + to survive upgrade without losing connectivity. +*/ +static void maybe_convert_charset(const char **user, const char **password, + const char **database, const char *csname) +{ + if (GetACP() != CP_UTF8 || !strncmp(csname, "utf8", 4)) + return; + static char bufs[3][CNV_BUFSIZE]; + const char **from[]= {user, password, database}; + CHARSET_INFO *cs= get_charset_by_csname(csname, MY_CS_PRIMARY, + MYF(MY_UTF8_IS_UTF8MB3 | MY_WME)); + if (!cs) + return; + for (int i= 0; i < 3; i++) + { + const char *str= *from[i]; + if (!str) + continue; + uint errors; + uint len= my_convert(bufs[i], CNV_BUFSIZE, cs, str, (uint32) strlen(str), + &my_charset_utf8mb4_bin, &errors); + bufs[i][len]= 0; + *from[i]= bufs[i]; + } +} +#endif + /* set connection-specific options and call mysql_real_connect */ @@ -1473,6 +1513,10 @@ static bool do_connect(MYSQL *mysql, const char *host, const char *user, mysql_options(mysql, MYSQL_OPT_CONNECT_ATTR_RESET, 0); mysql_options4(mysql, MYSQL_OPT_CONNECT_ATTR_ADD, "program_name", "mysql"); +#ifdef _WIN32 + maybe_convert_charset(&user, &password, &database,default_charset); +#endif + return mysql_real_connect(mysql, host, user, password, database, opt_mysql_port, opt_mysql_unix_port, flags); } diff --git a/mysql-test/main/charset_client_win_utf8mb4.result b/mysql-test/main/charset_client_win_utf8mb4.result index f7b5d376f9a..9bbf751e45c 100644 --- a/mysql-test/main/charset_client_win_utf8mb4.result +++ b/mysql-test/main/charset_client_win_utf8mb4.result @@ -1,2 +1,6 @@ @@character_set_client utf8mb4 +ERROR 1045 (28000): Access denied for user 'u'@'localhost' (using password: YES) +2 +2 +DROP user u; diff --git a/mysql-test/main/charset_client_win_utf8mb4.test b/mysql-test/main/charset_client_win_utf8mb4.test index 2baf0d7c050..e08afa250bd 100644 --- a/mysql-test/main/charset_client_win_utf8mb4.test +++ b/mysql-test/main/charset_client_win_utf8mb4.test @@ -1,3 +1,22 @@ --source include/windows.inc --source include/check_utf8_cli.inc --exec $MYSQL --default-character-set=auto -e "select @@character_set_client" + +# Test that a user with old, non-UTF8 password can still connect +# by setting setting non-auto --default-character-set +# This is important for backward compatibility + +# Emulate creating password in an interactive client session, with older clients +# which communicates with the server using with something like cp850 + +exec chcp 850 > NUL && echo CREATE USER 'u' IDENTIFIED by 'ü' | $MYSQL --default-character-set=cp850; + + +# Can't connect with UTF8 +--error 1 +exec $MYSQL --default-character-set=auto --user=u --password=ü -e "select 1" 2>&1; + +# Can connect with tweaked --default-character-set +exec $MYSQL --default-character-set=cp850 --user=u --password=ü -e "select 2"; +DROP user u; + -- cgit v1.2.1 From 2e48fbe3f5e84ebb8a2de4f4fc0448d648d25c0c Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Jan 2022 17:32:53 +0100 Subject: MDEV-27525 Invalid (non-UTF8) characters found for option 'plugin_dir' Two Problems 1. Upgrade wizard failed to retrieve path to service executable, if it contained non-ASCII. Fixed by setlocale(LC_ALL, "en_US.UTF8"), which was missing in upgrade wizard 2.mysql_upgrade_service only updated (converted to UTF8) the server's sections leaving client's as-is Corrected typo. 3. Fixed assertion in my_getopt, turns out to be too strict. --- mysys/my_getopt.c | 5 ----- sql/upgrade_conf_file.cc | 2 +- win/upgrade_wizard/upgradeDlg.cpp | 6 ++++++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 6e9c6334620..653fe844093 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -163,11 +163,6 @@ static void validate_value(const char *key, const char *value, } else { - /* - Should never happen, non-UTF8 can be read from option's - file only. - */ - DBUG_ASSERT(0); my_getopt_error_reporter( WARNING_LEVEL, "%s: invalid (non-UTF8) characters for option %s", my_progname, key); diff --git a/sql/upgrade_conf_file.cc b/sql/upgrade_conf_file.cc index 9a19dd8b8a2..0d7bc603468 100644 --- a/sql/upgrade_conf_file.cc +++ b/sql/upgrade_conf_file.cc @@ -288,7 +288,7 @@ static bool is_mariadb_section(const char *name, bool *is_server) if (*is_server= !strcmp(section_name, name)) break; - return *is_server; + return true; } diff --git a/win/upgrade_wizard/upgradeDlg.cpp b/win/upgrade_wizard/upgradeDlg.cpp index 7aae4890b51..10a1787c231 100644 --- a/win/upgrade_wizard/upgradeDlg.cpp +++ b/win/upgrade_wizard/upgradeDlg.cpp @@ -15,6 +15,7 @@ #include #include +#include using namespace std; @@ -271,6 +272,11 @@ BOOL CUpgradeDlg::OnInitDialog() m_Progress.ShowWindow(SW_HIDE); m_Ok.EnableWindow(FALSE); + if (GetACP() == CP_UTF8) + { + /* Required for mbstowcs, used in some functions.*/ + setlocale(LC_ALL, "en_US.UTF8"); + } PopulateServicesList(); return TRUE; // return TRUE unless you set the focus to a control } -- cgit v1.2.1