[erts] Option 'local' of term_to_binary/2 and term_to_iovec/2

Introduce a local external term format with an unspecified encoding. This
external term format is used for supporting the 'local' option of
term_to_binary/2 and term_to_iovec/2. Terms encoded on this format is only
to be decoded by the same runtime system instance that encoded it.

4 files changed, 511 insertions, 2 deletions

diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index b8683385cf..cf63cce3d7 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -77,7 +77,8 @@
 	 error_after_yield/1, cmp_old_impl/1,
          t2b_system_limit/1,
          term_to_iovec/1,
-         is_binary_test/1]).
+         is_binary_test/1,
+         local_ext/1]).
 
 %% Internal exports.
 -export([sleeper/0,trapping_loop/4]).
@@ -106,7 +107,8 @@ all() ->
      bit_sized_binary_sizes, otp_6817, otp_8117, deep,
      robustness, otp_8180, trapping, large,
      error_after_yield, cmp_old_impl,
-     is_binary_test].
+     is_binary_test,
+     local_ext].
 
 groups() -> 
     [
@@ -2348,3 +2350,177 @@ list2bitstrlist([X0, X1, X2, X3, X4, X5 | Xs], Acc) when is_integer(X0), 0 =< X0
     list2bitstrlist(Xs, NewAcc);
 list2bitstrlist([X | Xs], Acc) ->
     list2bitstrlist(Xs, [Acc,X]).
+
+local_ext(Config) when is_list(Config) ->
+    SDrv = send_term_local_drv,
+    CDrv = call_local_drv,
+    DataDir = proplists:get_value(data_dir, Config),
+    PrivDir = proplists:get_value(priv_dir, Config),
+    FileName = filename:join(PrivDir, "local_ext.data"),
+    Args = ["-setcookie", atom_to_list(erlang:get_cookie()),
+            "-pa", filename:dirname(code:which(?MODULE))],
+    {ok, Peer1, _} = peer:start_link(#{connection => 0, args => Args}),
+    {ok, Peer2, _} = peer:start_link(#{connection => 0, args => Args}),
+    LongNames = net_kernel:longnames(),
+    DynStartOpts = #{name_domain => if LongNames -> longnames;
+                                       true -> shortnames
+                                    end},
+    ExternalPid = self(),
+    ExternalRef = make_ref(),
+    ExternalPort = hd(erlang:ports()),
+    EncDecLocal = fun () ->
+                          erl_ddll:start(),
+                          ok = erl_ddll:load_driver(DataDir, SDrv),
+                          SPort = open_port({spawn, SDrv}, []),
+                          ok = erl_ddll:load_driver(DataDir, CDrv),
+                          CPort = open_port({spawn, CDrv}, []),
+                          false = erlang:is_alive(),
+                          nonode@nohost = node(),
+                          LocalPid = self(),
+                          LocalRef = make_ref(),
+                          LocalPort = hd(erlang:ports()),
+                          Bin1 = <<4711:800>>,
+                          Bin2 = <<4711:703>>,
+                          Bin3 = <<4711:600>>,
+                          Terms = [
+                                   LocalPid,
+                                   ExternalPid,
+                                   LocalRef,
+                                   ExternalRef,
+                                   LocalPort,
+                                   ExternalPort,
+                                   [LocalPid, Bin1, ExternalPid, LocalRef, Bin2,
+                                    ExternalRef, Bin3, Bin2, LocalPort,
+                                    ExternalPort],
+                                   "hej",
+                                   [],
+                                   {processes(), Bin3, erlang:ports(), Bin3},
+                                   #{pid => LocalPid, ref => LocalRef, port => LocalPort}
+                                  ],
+                          {ok, FD} = file:open(FileName, [write]),
+                          ETs = lists:map(fun (Term) ->
+                                                  {enc_local(FD, Term), Term}
+                                          end, Terms),
+                          ok = file:close(FD),
+                          CheckET = fun ({LExt, Term}) ->
+                                            Term = binary_to_term(LExt),
+                                            SPort ! {self(), {command, LExt}},
+                                            receive
+                                                {SPort, Reply} ->
+                                                    Term = Reply
+                                            end
+                                    end,
+                          lists:foreach(CheckET, ETs),
+                          call_local_success(CPort, ETs),
+                          NodeName = peer:random_name(),
+                          {ok, _} = net_kernel:start(list_to_atom(NodeName),
+                                                     DynStartOpts),
+                          true = erlang:is_alive(),
+                          true = nonode@nohost /= node(),
+                          lists:foreach(CheckET, ETs),
+                          call_local_success(CPort, ETs),
+                          ok = net_kernel:stop(),
+                          false = erlang:is_alive(),
+                          nonode@nohost = node(),
+                          lists:foreach(CheckET, ETs),
+                          call_local_success(CPort, ETs),
+                          {ok, ExtList} = file:consult(FileName),
+                          lists:foreach(fun (Ext) when is_binary(Ext) ->
+                                                _ = binary_to_term(Ext),
+                                                SPort ! {self(), {command, Ext}},
+                                                receive
+                                                    {SPort, "bad_term_error"} ->
+                                                        error(bad_term_error);
+                                                    {SPort, _} ->
+                                                        ok
+                                                end
+                                        end,
+                                        ExtList),
+                          true = port_close(SPort),
+                          true = port_close(CPort),
+                          ok
+                  end,
+    ok = peer:call(Peer1, erlang, apply, [EncDecLocal, []]),
+    DecOthersLocal = fun () ->
+                             %% Verify that decoding of the terms encoded
+                             %% on local external format by the other runtime
+                             %% system instance fails on this runtime system
+                             %% instance...
+                             erl_ddll:start(),
+                             ok = erl_ddll:load_driver(DataDir, SDrv),
+                             SPort = open_port({spawn, SDrv}, []),
+                             ok = erl_ddll:load_driver(DataDir, CDrv),
+                             CPort = open_port({spawn, CDrv}, []),
+                             false = erlang:is_alive(),
+                             nonode@nohost = node(),
+                             {ok, ExtList} = file:consult(FileName),
+                             lists:foreach(fun (Ext) when is_binary(Ext) ->
+                                                   try
+                                                       Term = binary_to_term(Ext),
+                                                       error({successful_decode, Term})
+                                                   catch
+                                                       error:badarg ->
+                                                           ok
+                                                   end,
+                                                   SPort ! {self(), {command, Ext}},
+                                                   receive
+                                                       {SPort, Reply} ->
+                                                           "bad_term_error" = Reply
+                                                   end
+                                           end,
+                                           ExtList),
+                             call_local_fail(CPort, ExtList),
+                             true = port_close(SPort),
+                             true = port_close(CPort),
+                             ok
+                     end,
+    ok = peer:call(Peer2, erlang, apply, [DecOthersLocal, []]),
+    peer:stop(Peer1),
+    peer:stop(Peer2),
+    ok.
+
+enc_local(FD, Term) ->
+    Ext = term_to_binary(Term, [local]),
+    Ext = iolist_to_binary(term_to_iovec(Term, [local])),
+    Term = binary_to_term(Ext),
+    io:format(FD, "~p.~n", [Ext]),
+    Ext.
+
+call_local_success(Port, []) ->
+    ok;
+call_local_success(Port, [{Lext1, T1}]) ->
+    Me = self(),
+    Ref = make_ref(),
+    Term =  {term_to_binary(Me), Lext1, term_to_binary(Ref)},
+    {call_result, Me, 4711, T1, 17, Ref, "end_of_data"} = erlang:port_call(Port, 0, Term),
+    ok;
+call_local_success(Port, [{Lext1, T1}, {Lext3, T3} | Rest]) ->
+    Me = self(),
+    Term =  {Lext1, term_to_binary(Me), Lext3},
+    {call_result, T1, 4711, Me, 17, T3, "end_of_data"} = erlang:port_call(Port, 0, Term),
+    call_local_success(Port, Rest).
+
+call_local_fail(Port, []) ->
+    ok;
+call_local_fail(Port, [Lext1]) ->
+    Me = self(),
+    Ref = make_ref(),
+    Term =  {term_to_binary(Me), Lext1, term_to_binary(Ref)},
+    try
+        erlang:port_call(Port, 0, Term),
+        error(unexpected_port_call_success)
+    catch
+        error:badarg ->
+            ok
+    end;
+call_local_fail(Port, [Lext1, Lext3 | Rest]) ->
+    Me = self(),
+    Term =  {Lext1, term_to_binary(Me), Lext3},
+    try
+        erlang:port_call(Port, 0, Term),
+        error(unexpected_port_call_success)
+    catch
+        error:badarg ->
+            ok
+    end,
+    call_local_fail(Port, Rest).
diff --git a/erts/emulator/test/binary_SUITE_data/Makefile.src b/erts/emulator/test/binary_SUITE_data/Makefile.src
new file mode 100644
index 0000000000..541dd6c1ad
--- /dev/null
+++ b/erts/emulator/test/binary_SUITE_data/Makefile.src
@@ -0,0 +1,33 @@
+#
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2023. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# %CopyrightEnd%
+#
+
+include @erl_interface_mk_include@
+
+CC = @CC@
+LD = @LD@
+CFLAGS = @CFLAGS@ -I@erl_include@ @DEFS@
+CROSSLDFLAGS = @CROSSLDFLAGS@
+
+SHLIB_EXTRA_CFLAGS = @EI_CFLAGS@ -I@erl_interface_include@ 
+SHLIB_EXTRA_LDLIBS = @erl_interface_eilib@
+
+all: send_term_local_drv@dll@ call_local_drv@dll@
+
+@SHLIB_RULES@
diff --git a/erts/emulator/test/binary_SUITE_data/call_local_drv.c b/erts/emulator/test/binary_SUITE_data/call_local_drv.c
new file mode 100644
index 0000000000..8db6c1c772
--- /dev/null
+++ b/erts/emulator/test/binary_SUITE_data/call_local_drv.c
@@ -0,0 +1,204 @@
+/* ``Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
+ * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
+ * AB. All Rights Reserved.''
+ * 
+ *     $Id$
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include "erl_driver.h"
+#include "ei.h"
+
+static ErlDrvSSizeT call(ErlDrvData drv_data,
+			 unsigned int command,
+			 char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen,
+			 unsigned int *flags);
+
+static ErlDrvEntry call_local_drv_entry = { 
+    NULL /* init */,
+    NULL /* start */,
+    NULL /* stop */,
+    NULL /* output */,
+    NULL /* ready_input */,
+    NULL /* ready_output */,
+    "call_local_drv",
+    NULL /* finish */,
+    NULL /* handle */,
+    NULL /* control */,
+    NULL /* timeout */,
+    NULL /* outputv */,
+    NULL /* ready_async */,
+    NULL /* flush */,
+    call,
+    NULL /* event */,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    ERL_DRV_FLAG_USE_PORT_LOCKING,
+    NULL /* handle2 */,
+    NULL /* handle_monitor */
+};
+
+DRIVER_INIT(call_local_drv)
+{
+    return &call_local_drv_entry;
+}
+
+
+static ErlDrvSSizeT call(ErlDrvData drv_data,
+			 unsigned int command,
+			 char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen,
+			 unsigned int *flags)
+{
+    ei_x_buff xbuf;
+    void *bin1 = NULL, *bin2 = NULL, *bin3 = NULL;
+    char *lext1, *lext2, *lext3;
+    int vsn, arity, type, ix, lix, res, err, size;
+    long size1, size2, size3;
+    ErlDrvSSizeT ret_size = (ErlDrvSSizeT) ERL_DRV_ERROR_GENERAL;
+
+    xbuf.buff = NULL;
+
+    ei_init();
+
+    ix = 0;
+    res = ei_decode_version(buf, &ix, &vsn);
+    if (res != 0 || vsn != 131)
+        goto error;
+
+    res = ei_decode_tuple_header(buf, &ix, &arity);
+    if (res != 0)
+        goto error;
+
+    /* External term 1 */
+    res = ei_get_type(buf, &ix, &type, &size);
+    if (res != 0 && type != ERL_BINARY_EXT)
+        goto error;
+
+    size1 = size;
+    bin1 = driver_alloc(size1);
+
+    res = ei_decode_binary(buf, &ix, bin1, &size1);
+    if (res != 0 && type != ERL_BINARY_EXT)
+        goto error;
+
+    lext1 = bin1;
+    lix = 0;
+    res = ei_decode_version(lext1, &lix, &vsn);
+    if (res != 0 || vsn != 131)
+        goto error;
+    lext1 += lix;
+    size1 -= lix;
+
+    /* External term 2 */
+    res = ei_get_type(buf, &ix, &type, &size);
+    if (res != 0 && type != ERL_BINARY_EXT)
+        goto error;
+
+    size2 = size;
+    bin2 = driver_alloc(size2);
+
+    res = ei_decode_binary(buf, &ix, bin2, &size2);
+    if (res != 0 && type != ERL_BINARY_EXT)
+        goto error;
+
+    lext2 = bin2;
+    lix = 0;
+    res = ei_decode_version(lext2, &lix, &vsn);
+    if (res != 0 || vsn != 131)
+        goto error;
+    lext2 += lix;
+    size2 -= lix;
+
+    /* External term 3 */
+    res = ei_get_type(buf, &ix, &type, &size);
+    if (res != 0 && type != ERL_BINARY_EXT)
+        goto error;
+
+    size3 = size;
+    bin3 = driver_alloc(size3);
+
+    res = ei_decode_binary(buf, &ix, bin3, &size3);
+    if (res != 0 && type != ERL_BINARY_EXT)
+        goto error;
+
+    lext3 = bin3;
+    lix = 0;
+    res = ei_decode_version(lext3, &lix, &vsn);
+    if (res != 0 || vsn != 131)
+        goto error;
+    lext3 += lix;
+    size3 -= lix;
+
+    /* encode result */
+
+    res = ei_x_new_with_version(&xbuf);
+    if (res != 0)
+        goto error;
+
+    res = ei_x_encode_tuple_header(&xbuf, 7);
+    if (res != 0)
+        goto error;
+
+    res = ei_x_encode_atom(&xbuf, "call_result");
+    if (res != 0)
+        goto error;
+
+    res = ei_x_append_buf(&xbuf, lext1, size1);
+    if (res != 0)
+        goto error;
+
+    res = ei_x_encode_long(&xbuf, 4711);
+    if (res != 0)
+        goto error;
+
+    res = ei_x_append_buf(&xbuf, lext2, size2);
+    if (res != 0)
+        goto error;
+
+    res = ei_x_encode_long(&xbuf, 17);
+    if (res != 0)
+        goto error;
+
+    res = ei_x_append_buf(&xbuf, lext3, size3);
+    if (res != 0)
+        goto error;
+
+    res = ei_x_encode_string(&xbuf, "end_of_data");
+    if (res != 0)
+        goto error;
+
+    /* success */
+    ret_size = xbuf.index;
+    *rbuf = driver_alloc(ret_size);
+    memcpy((void *) *rbuf, (void *) xbuf.buff, ret_size);
+
+error:
+
+    if (bin1)
+        driver_free(bin1);
+    if (bin2)
+        driver_free(bin2);
+    if (bin3)
+        driver_free(bin3);
+
+    if (xbuf.buff)
+        ei_x_free(&xbuf);
+
+    return ret_size;
+}
diff --git a/erts/emulator/test/binary_SUITE_data/send_term_local_drv.c b/erts/emulator/test/binary_SUITE_data/send_term_local_drv.c
new file mode 100644
index 0000000000..a5335d4a09
--- /dev/null
+++ b/erts/emulator/test/binary_SUITE_data/send_term_local_drv.c
@@ -0,0 +1,96 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2023. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "erl_driver.h"
+#include <string.h>
+
+static void stop(ErlDrvData drv_data);
+static ErlDrvData start(ErlDrvPort port,
+			char *command);
+static void output(ErlDrvData drv_data,
+		   char *buf, ErlDrvSizeT len);
+
+static ErlDrvEntry send_term_local_drv_entry = { 
+    NULL /* init */,
+    start,
+    stop,
+    output,
+    NULL /* ready_input */,
+    NULL /* ready_output */,
+    "send_term_local_drv",
+    NULL /* finish */,
+    NULL /* handle */,
+    NULL /* control */,
+    NULL /* timeout */,
+    NULL /* outputv */,
+    NULL /* ready_async */,
+    NULL /* flush */,
+    NULL /* call */,
+    NULL /* event */,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    ERL_DRV_FLAG_USE_PORT_LOCKING,
+    NULL /* handle2 */,
+    NULL /* handle_monitor */
+};
+
+DRIVER_INIT(send_term_local_drv)
+{
+    return &send_term_local_drv_entry;
+}
+
+static void stop(ErlDrvData drv_data)
+{
+}
+
+static ErlDrvData start(ErlDrvPort port,
+			char *command)
+{
+
+    return (ErlDrvData) port;
+}
+
+static void output(ErlDrvData drv_data,
+		   char *buf, ErlDrvSizeT len)
+{
+    ErlDrvPort port = (ErlDrvPort) drv_data;
+    ErlDrvTermData term_port = driver_mk_port(port);
+    ErlDrvTermData caller = driver_caller(port);
+    int res;
+    ErlDrvTermData spec[] = {
+	ERL_DRV_PORT, term_port,
+	ERL_DRV_EXT2TERM, (ErlDrvTermData) buf, len,
+	ERL_DRV_TUPLE, 2
+    };
+    if (0 >= erl_drv_send_term(term_port, caller,
+                               spec, sizeof(spec)/sizeof(spec[0]))) {
+        char *bad_term = "bad_term_error";
+        ErlDrvTermData spec[] = {
+            ERL_DRV_PORT, term_port,
+            ERL_DRV_STRING, (ErlDrvTermData) bad_term, strlen(bad_term),
+            ERL_DRV_TUPLE, 2
+        };
+        if (0 >= erl_drv_send_term(term_port, caller, spec,
+                                   sizeof(spec)/sizeof(spec[0]))) {
+            driver_failure_atom(port, "failed_to_bad_term_error");
+        }
+    }
+}