re:run(): ensure badarg on illegal utf8 in subject

Also handle all other returns from erts_pcre_exec() properly
regarding reductions.

3 files changed, 115 insertions, 24 deletions

diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index ad124fd979..6382b2bdb6 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1353,31 +1353,69 @@ handle_iolist:
     rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, 
 			slength, startoffset, 
 			options, restart.ovector, ovsize);
-
-    if (rc == PCRE_ERROR_BADENDIANNESS || rc == PCRE_ERROR_BADMAGIC) {
-	cleanup_restart_context(&restart);
-	BIF_ERROR(p,BADARG);
+    if (rc < 0) {
+        switch (rc) {
+            /* No match... */
+        case PCRE_ERROR_NOMATCH:
+        case PCRE_ERROR_MATCHLIMIT:
+        case PCRE_ERROR_RECURSIONLIMIT:
+            break;
+
+            /* Yield... */
+        case PCRE_ERROR_LOOP_LIMIT: {
+            /* Trap */
+            Binary *mbp = erts_create_magic_binary(sizeof(RestartContext),
+                                                   cleanup_restart_context_bin);
+            RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp);
+            Eterm magic_ref;
+            Eterm *hp;
+            ASSERT(loop_count != 0xFFFFFFFF);
+            BUMP_REDS(p, loop_count / LOOP_FACTOR);
+            memcpy(restartp,&restart,sizeof(RestartContext));
+            ERTS_VBUMP_ALL_REDS(p);
+            hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+            magic_ref = erts_mk_magic_ref(&hp, &MSO(p), mbp);
+            BIF_TRAP3(&re_exec_trap_export, 
+                      p,
+                      arg1,
+                      arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */,
+                      magic_ref);
+        }
+
+            /* Recursive loop detected in pattern... */
+        case PCRE_ERROR_RECURSELOOP:
+#if 1
+            loop_count = CONTEXT_REDS*LOOP_FACTOR; /* Unknown amount of work done... */
+            break; /* nomatch for backwards compatibility reasons for now... */
+#else
+            BUMP_ALL_REDS(p); /* Unknown amount of work done... */
+            cleanup_restart_context(&restart);
+            BIF_ERROR(p, BADARG);
+#endif
+            
+            /* Bad utf8 in subject... */
+        case PCRE_ERROR_SHORTUTF8:
+        case PCRE_ERROR_BADUTF8:
+        case PCRE_ERROR_BADUTF8_OFFSET:
+            BUMP_ALL_REDS(p); /* Unknown amount of work done... */
+            /* Fall through for badarg... */
+            
+            /* Bad pre-compiled regexp... */
+        case PCRE_ERROR_BADMAGIC:
+        case PCRE_ERROR_BADENDIANNESS:
+            cleanup_restart_context(&restart);
+            BIF_ERROR(p, BADARG);
+            
+        default:
+            /* Something unexpected happened... */
+            ASSERT(! "Unexpected erts_pcre_exec() result");
+            cleanup_restart_context(&restart);
+            BIF_ERROR(p, EXC_INTERNAL_ERROR);
+        }
     }
     
     ASSERT(loop_count != 0xFFFFFFFF);
     BUMP_REDS(p, loop_count / LOOP_FACTOR);
-    if (rc == PCRE_ERROR_LOOP_LIMIT) {
-	/* Trap */
-	Binary *mbp = erts_create_magic_binary(sizeof(RestartContext),
-					       cleanup_restart_context_bin);
-	RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp);
-	Eterm magic_ref;
-	Eterm *hp;
-	memcpy(restartp,&restart,sizeof(RestartContext));
-	BUMP_ALL_REDS(p);
-	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
-	magic_ref = erts_mk_magic_ref(&hp, &MSO(p), mbp);
-	BIF_TRAP3(&re_exec_trap_export, 
-		  p,
-		  arg1,
-		  arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */,
-		  magic_ref);
-    }
 
     res = build_exec_return(p, rc, &restart, arg1);
  
diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src
index 3c449d3cb9..618aa1caeb 100644
--- a/lib/stdlib/src/stdlib.app.src
+++ b/lib/stdlib/src/stdlib.app.src
@@ -107,7 +107,7 @@
                dets]},
   {applications, [kernel]},
   {env, []},
-  {runtime_dependencies, ["sasl-3.0","kernel-5.0","erts-9.0","crypto-3.3",
+  {runtime_dependencies, ["sasl-3.0","kernel-5.0","erts-@OTP-16553@","crypto-3.3",
 			  "compiler-5.0"]}
 ]}.
 
diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl
index 7b82647416..8bfd8a5732 100644
--- a/lib/stdlib/test/re_SUITE.erl
+++ b/lib/stdlib/test/re_SUITE.erl
@@ -28,7 +28,8 @@
 	 pcre_compile_workspace_overflow/1,re_infinite_loop/1, 
 	 re_backwards_accented/1,opt_dupnames/1,opt_all_names/1,inspect/1,
 	 opt_no_start_optimize/1,opt_never_utf/1,opt_ucp/1,
-	 match_limit/1,sub_binaries/1,copt/1]).
+	 match_limit/1,sub_binaries/1,copt/1,
+         bad_utf8_subject/1]).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("kernel/include/file.hrl").
@@ -45,7 +46,8 @@ all() ->
      pcre_compile_workspace_overflow, re_infinite_loop, 
      re_backwards_accented, opt_dupnames, opt_all_names, 
      inspect, opt_no_start_optimize,opt_never_utf,opt_ucp,
-     match_limit, sub_binaries, re_version].
+     match_limit, sub_binaries, re_version,
+     bad_utf8_subject].
 
 groups() -> 
     [].
@@ -904,3 +906,54 @@ sub_binaries(Config) when is_list(Config) ->
     {match,[D]}=re:run(Bin,"a(.+)$",[{capture,[1],binary}]),
     4096 = binary:referenced_byte_size(D),
     ok.
+
+bad_utf8_subject(Config) when is_list(Config) ->
+    %% OTP-16553: re:run() did not badarg
+    %% if both pattern and subject was binaries
+    %% even though subject contained illegal
+    %% utf8...
+
+    nomatch = re:run(<<255,255,255>>, <<"a">>, []),
+    nomatch = re:run(<<255,255,255>>, "a", []),
+    nomatch = re:run(<<"aaa">>, <<255>>, []),
+    nomatch = re:run(<<"aaa">>, [255], []),
+    {match,[{0,1}]} = re:run(<<255,255,255>>, <<255>>, []),
+    {match,[{0,1}]} = re:run(<<255,255,255>>, [255], []),
+    %% Badarg on illegal utf8 in subject as of OTP 23...
+    try
+        re:run(<<255,255,255>>, <<"a">>, [unicode]),
+        error(unexpected)
+    catch
+        error:badarg ->
+            ok
+    end,
+    try
+        re:run(<<255,255,255>>, "a", [unicode]),
+        error(unexpected)
+    catch
+        error:badarg ->
+            ok
+    end,
+    try
+        re:run(<<"aaa">>, <<255>>, [unicode]),
+        error(unexpected)
+    catch
+        error:badarg ->
+            ok
+    end,
+    nomatch = re:run(<<"aaa">>, [255], [unicode]),
+    try
+        re:run(<<255,255,255>>, <<255>>, [unicode]),
+        error(unexpected)
+    catch
+        error:badarg ->
+            ok
+    end,
+    try
+        re:run(<<255,255,255>>, [255], [unicode]),
+        error(unexpected)
+    catch
+        error:badarg ->
+            ok
+    end.
+