mysqltest - use my_popen()/my_fgets() wrappers for popen()/fgets()bb-10.2-wlad-popen

On Windows, CRT version of popen/fgets() has a bug, such that
fgets()sometimes loses end of line when reading from pipe
(details in MDEV-9409)

The workaround is to use binary mode for the FILE* returned by popen(),
and to strip '\r' from the end of line manually.

The workaround was used previously for "exec", but now wrapper functions
are used also for "perl" snippets.

2 files changed, 69 insertions, 55 deletions

diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index 05113f02a44..6e844ce1799 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -836,6 +836,47 @@ void revert_properties();
 static void handle_no_active_connection(struct st_command* command, 
   struct st_connection *cn, DYNAMIC_STRING *ds);
 
+
+/* Wrapper for fgets.Strips \r off newlines on Windows.
+   Should be used with together with my_popen().
+*/
+static char *my_fgets(char * s, int n, FILE * stream, int *len)
+{
+  char *buf = fgets(s, n, stream);
+  if (!buf)
+  {
+    *len= 0;
+    return buf;
+  }
+
+  *len = (int)strlen(buf);
+#ifdef _WIN32
+  /* Strip '\r' off newlines. */
+  if (*len > 1 && buf[*len - 2] == '\r' && buf[*len - 1] == '\n')
+  {
+    buf[*len - 2]= '\n';
+    buf[*len - 1]= 0;
+    (*len)--;
+  }
+#endif
+  return buf;
+}
+
+/*
+  Wrapper for popen().
+  On Windows, uses binary mode to workaround
+  C runtime bug mentioned in MDEV-9409
+*/
+static FILE* my_popen(const char *cmd, const char *mode)
+{
+  FILE *f= popen(cmd, mode);
+#ifdef _WIN32
+  if (f)
+    _setmode(fileno(f), O_BINARY);
+#endif
+  return f;
+}
+
 #ifdef EMBEDDED_LIBRARY
 
 #define EMB_SEND_QUERY 1
@@ -1785,19 +1826,20 @@ static int run_command(char* cmd,
   DBUG_ENTER("run_command");
   DBUG_PRINT("enter", ("cmd: %s", cmd));
 
-  if (!(res_file= popen(cmd, "r")))
+  if (!(res_file= my_popen(cmd, "r")))
   {
     report_or_die("popen(\"%s\", \"r\") failed", cmd);
     DBUG_RETURN(-1);
   }
 
-  while (fgets(buf, sizeof(buf), res_file))
+  int len;
+  while (my_fgets(buf, sizeof(buf), res_file, &len))
   {
     DBUG_PRINT("info", ("buf: %s", buf));
     if(ds_res)
     {
       /* Save the output of this command in the supplied string */
-      dynstr_append(ds_res, buf);
+      dynstr_append_mem(ds_res, buf,len);
     }
     else
     {
@@ -1886,14 +1928,15 @@ static int diff_check(const char *diff_name)
 
   my_snprintf(buf, sizeof(buf), "%s -v", diff_name);
 
-  if (!(res_file= popen(buf, "r")))
+  if (!(res_file= my_popen(buf, "r")))
     die("popen(\"%s\", \"r\") failed", buf);
 
   /*
     if diff is not present, nothing will be in stdout to increment
     have_diff
   */
-  if (fgets(buf, sizeof(buf), res_file))
+  int len;
+  if (my_fgets(buf, sizeof(buf), res_file, &len))
     have_diff= 1;
 
   pclose(res_file);
@@ -3246,18 +3289,6 @@ void free_tmp_sh_file()
 #endif
 
 
-FILE* my_popen(DYNAMIC_STRING *ds_cmd, const char *mode)
-{
-#if defined _WIN32 && defined USE_CYGWIN
-  /* Dump the command into a sh script file and execute with popen */
-  str_to_file(tmp_sh_name, ds_cmd->str, ds_cmd->length);
-  return popen(tmp_sh_cmd, mode);
-#else
-  return popen(ds_cmd->str, mode);
-#endif
-}
-
-
 static void init_builtin_echo(void)
 {
 #ifdef _WIN32
@@ -3392,7 +3423,7 @@ void do_exec(struct st_command *command)
   DBUG_PRINT("info", ("Executing '%s' as '%s'",
                       command->first_argument, ds_cmd.str));
 
-  if (!(res_file= my_popen(&ds_cmd, "r")))
+  if (!(res_file= my_popen(ds_cmd.str, "r")))
   {
     dynstr_free(&ds_cmd);
     if (command->abort_on_error)
@@ -3406,24 +3437,9 @@ void do_exec(struct st_command *command)
     init_dynamic_string(&ds_sorted, "", 1024, 1024);
     ds_result= &ds_sorted;
   }
-
-#ifdef _WIN32
-   /* Workaround for CRT bug, MDEV-9409 */
-  _setmode(fileno(res_file), O_BINARY);
-#endif
-
-  while (fgets(buf, sizeof(buf), res_file))
+  int len;
+  while (my_fgets(buf, sizeof(buf), res_file,&len))
   {
-    int len = (int)strlen(buf);
-#ifdef _WIN32
-    /* Strip '\r' off newlines. */
-    if (len > 1 && buf[len-2] == '\r' && buf[len-1] == '\n')
-    {
-      buf[len-2] = '\n';
-      buf[len-1] = 0;
-      len--;
-    }
-#endif
     replace_dynstr_append_mem(ds_result, buf, len);
   }
   error= pclose(res_file);
@@ -4685,7 +4701,7 @@ void do_perl(struct st_command *command)
     /* Format the "perl <filename>" command */
     my_snprintf(buf, sizeof(buf), "perl %s", temp_file_path);
 
-    if (!(res_file= popen(buf, "r")))
+    if (!(res_file= my_popen(buf, "r")))
     {
       if (command->abort_on_error)
         die("popen(\"%s\", \"r\") failed", buf);
@@ -4693,16 +4709,17 @@ void do_perl(struct st_command *command)
       DBUG_VOID_RETURN;
     }
 
-    while (fgets(buf, sizeof(buf), res_file))
+    int len;
+    while (my_fgets(buf, sizeof(buf), res_file,&len))
     {
       if (disable_result_log)
       {
-	buf[strlen(buf)-1]=0;
-	DBUG_PRINT("exec_result",("%s", buf));
+        buf[len - 1] = 0;
+        DBUG_PRINT("exec_result", ("%s", buf));
       }
       else
       {
-	replace_dynstr_append(&ds_res, buf);
+        replace_dynstr_append_mem(&ds_res, buf, len);
       }
     }
     error= pclose(res_file);
diff --git a/mysql-test/r/mysqld--help,win.rdiff b/mysql-test/r/mysqld--help,win.rdiff
index 4e46a9da285..ac0f75b94eb 100644
--- a/mysql-test/r/mysqld--help,win.rdiff
+++ b/mysql-test/r/mysqld--help,win.rdiff
@@ -16,7 +16,7 @@
   --net-buffer-length=# 
   Buffer length for TCP/IP and socket communication
   --net-read-timeout=# 
-@@ -956,6 +956,9 @@
+@@ -957,6 +957,9 @@
   characteristics (isolation level, read only/read
   write,snapshot - but not any work done / data modified
   within the transaction).
@@ -26,7 +26,7 @@
   --show-slave-auth-info 
   Show user and password in SHOW SLAVE HOSTS on this
   master.
-@@ -1068,6 +1071,10 @@
+@@ -1069,6 +1072,10 @@
   Log slow queries to given log file. Defaults logging to
   'hostname'-slow.log. Must be enabled to activate other
   slow log options
@@ -37,7 +37,7 @@
   --socket=name       Socket file to use for connection
   --sort-buffer-size=# 
   Each thread that needs to do a sort allocates a buffer of
-@@ -1086,6 +1093,7 @@
+@@ -1087,6 +1094,7 @@
   NO_ENGINE_SUBSTITUTION, PAD_CHAR_TO_FULL_LENGTH
   --stack-trace       Print a symbolic stack trace on failure
   (Defaults to on; use --skip-stack-trace to disable.)
@@ -45,22 +45,19 @@
   --standard-compliant-cte 
   Allow only CTEs compliant to SQL standard
   (Defaults to on; use --skip-standard-compliant-cte to disable.)
-@@ -1134,8 +1142,12 @@
+@@ -1135,6 +1143,11 @@
   --thread-pool-max-threads=# 
   Maximum allowed number of worker threads in the thread
   pool
-- --thread-pool-oversubscribe=# 
-- How many additional active worker threads in a group are
 + --thread-pool-min-threads=# 
 + Minimum number of threads in the thread pool.
 + --thread-pool-mode=name 
 + Chose implementation of the threadpool. One of: windows, 
 + generic
-+ --thread-pool-oversubscribe=#  How many additional active worker threads in a group are
+  --thread-pool-oversubscribe=# 
+  How many additional active worker threads in a group are
   allowed.
-  --thread-pool-prio-kickup-timer=# 
-  The number of milliseconds before a dequeued low-priority
-@@ -1172,8 +1184,8 @@
+@@ -1173,8 +1186,8 @@
   automatically convert it to an on-disk MyISAM or Aria
   table.
   -t, --tmpdir=name   Path for temporary files. Several paths may be specified,
@@ -71,7 +68,7 @@
   --transaction-alloc-block-size=# 
   Allocation block size for transactions to be stored in
   binary log
-@@ -1298,7 +1310,6 @@
+@@ -1299,7 +1312,6 @@
  key-cache-division-limit 100
  key-cache-file-hash-size 512
  key-cache-segments 0
@@ -79,7 +76,7 @@
  lc-messages en_US
  lc-messages-dir MYSQL_SHAREDIR/
  lc-time-names en_US
-@@ -1368,6 +1379,7 @@
+@@ -1369,6 +1381,7 @@
  myisam-stats-method NULLS_UNEQUAL
  myisam-use-mmap FALSE
  mysql56-temporal-format TRUE
@@ -87,7 +84,7 @@
  net-buffer-length 16384
  net-read-timeout 30
  net-retry-count 10
-@@ -1469,6 +1481,8 @@
+@@ -1470,6 +1483,8 @@
  session-track-state-change FALSE
  session-track-system-variables 
  session-track-transaction-info OFF
@@ -96,7 +93,7 @@
  show-slave-auth-info FALSE
  silent-startup FALSE
  skip-grant-tables TRUE
-@@ -1493,6 +1507,7 @@
+@@ -1494,6 +1509,7 @@
  slave-type-conversions 
  slow-launch-time 2
  slow-query-log FALSE
@@ -104,7 +101,7 @@
  sort-buffer-size 2097152
  sql-mode STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
  stack-trace TRUE
-@@ -1506,14 +1521,16 @@
+@@ -1507,14 +1523,16 @@
  sync-relay-log 10000
  sync-relay-log-info 10000
  sysdate-is-now FALSE