erts: Decompress module binaries before passing them to BIFs

11 files changed, 118 insertions, 43 deletions

diff --git a/bootstrap/lib/kernel/ebin/code.beam b/bootstrap/lib/kernel/ebin/code.beam
index eafa1d5aa3..4e618471d6 100644
--- a/bootstrap/lib/kernel/ebin/code.beam
+++ b/bootstrap/lib/kernel/ebin/code.beam
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 76c25c4f2a..9dadc07c36 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -160,7 +160,7 @@ static int read_iff_list(Eterm iff_list, Uint *res) {
 }
 
 BIF_RETTYPE
-code_get_chunk_2(BIF_ALIST_2)
+erts_internal_beamfile_chunk_2(BIF_ALIST_2)
 {
     Uint search_iff;
     IFF_Chunk chunk;
@@ -185,10 +185,27 @@ code_get_chunk_2(BIF_ALIST_2)
 
     if (iff_init(start, binary_size(Bin), &iff)) {
         if (iff_read_chunk(&iff, search_iff, &chunk) && chunk.size > 0) {
-            res = new_binary(BIF_P, (byte*)chunk.data, chunk.size);
-        }
+            Sint offset, bitoffs, bitsize;
+            Eterm real_bin;
+
+            ERTS_GET_REAL_BIN(Bin, real_bin, offset, bitoffs, bitsize);
+
+            if (bitoffs) {
+                res = new_binary(BIF_P, (byte*)chunk.data, chunk.size);
+            } else {
+                ErlSubBin *sb = (ErlSubBin*)HAlloc(BIF_P, ERL_SUB_BIN_SIZE);
 
-        iff_dtor(&iff);
+                sb->thing_word = HEADER_SUB_BIN;
+                sb->orig = real_bin;
+                sb->size = chunk.size;
+                sb->bitsize = 0;
+                sb->bitoffs = 0;
+                sb->offs = offset + (chunk.data - start);
+                sb->is_writable = 0;
+
+                res = make_binary(sb);
+            }
+        }
     }
 
     erts_free_aligned_binary_bytes(temp_alloc);
@@ -198,7 +215,7 @@ code_get_chunk_2(BIF_ALIST_2)
 
 /* Calculate the MD5 for a module. */
 BIF_RETTYPE
-code_module_md5_1(BIF_ALIST_1)
+erts_internal_beamfile_module_md5_1(BIF_ALIST_1)
 {
     byte* temp_alloc;
     byte* bytes;
@@ -280,7 +297,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
 }
 
 BIF_RETTYPE
-prepare_loading_2(BIF_ALIST_2)
+erts_internal_prepare_loading_2(BIF_ALIST_2)
 {
     byte* temp_alloc = NULL;
     byte* code;
diff --git a/erts/emulator/beam/beam_file.c b/erts/emulator/beam/beam_file.c
index 2e408b78e2..cbf9e23f23 100644
--- a/erts/emulator/beam/beam_file.c
+++ b/erts/emulator/beam/beam_file.c
@@ -669,8 +669,6 @@ static int parse_code_chunk(BeamFile *beam, IFF_Chunk *chunk) {
     return 1;
 }
 
-ErlDrvBinary *erts_gzinflate_buffer(char*, int);
-
 static int read_beam_chunks(const IFF_File *file,
                             int count, const Uint *ids,
                             IFF_Chunk *chunks) {
@@ -960,8 +958,6 @@ void beamfile_free(BeamFile *beam) {
     if (beam->dynamic_literals.entries) {
         beamfile_literal_dtor(&beam->dynamic_literals);
     }
-
-    iff_dtor(&beam->iff);
 }
 
 Sint beamfile_add_literal(BeamFile *beam, Eterm term) {
@@ -1085,18 +1081,7 @@ int iff_init(const byte *data, size_t size, IFF_File *iff) {
     beamreader_init(data, size, &reader);
 
     LoadAssert(beamreader_read_i32(&reader, &form_id));
-
-    /* Decompress the module if necessary. */
-    if (form_id != MakeIffId('F', 'O', 'R', '1')) {
-        ErlDrvBinary *bin = erts_gzinflate_buffer((char*)data, size);
-
-        LoadAssert(bin);
-        iff->decompressed_data = bin;
-
-        beamreader_init((const byte*)bin->orig_bytes, bin->orig_size, &reader);
-        LoadAssert(beamreader_read_i32(&reader, &form_id));
-        LoadAssert(form_id == MakeIffId('F', 'O', 'R', '1'));
-    }
+    LoadAssert(form_id == MakeIffId('F', 'O', 'R', '1'));
 
     LoadAssert(beamreader_read_i32(&reader, &form_size));
     LoadAssert(beamreader_read_bytes(&reader, form_size, &real_data));
@@ -1107,13 +1092,6 @@ int iff_init(const byte *data, size_t size, IFF_File *iff) {
     return 1;
 }
 
-void iff_dtor(IFF_File *iff) {
-    if (iff->decompressed_data) {
-        driver_free_binary(iff->decompressed_data);
-        iff->decompressed_data = NULL;
-    }
-}
-
 int iff_read_chunk(IFF_File *iff, Uint id, IFF_Chunk *chunk)
 {
     return read_beam_chunks(iff, 1, &id, chunk);
diff --git a/erts/emulator/beam/beam_file.h b/erts/emulator/beam/beam_file.h
index 3c49db41ea..fe68447b8e 100644
--- a/erts/emulator/beam/beam_file.h
+++ b/erts/emulator/beam/beam_file.h
@@ -32,8 +32,6 @@
   (((Uint) (a) << 24) | ((Uint) (b) << 16) | ((Uint) (c) << 8) | (Uint) (d))
 
 typedef struct {
-    struct erl_drv_binary *decompressed_data;
-
     Sint32 form_id;
 
     Sint32 size;
@@ -54,7 +52,6 @@ int iff_init(const byte *data, size_t size, IFF_File *iff);
  *
  * @return 1 on success, 0 on failure */
 int iff_read_chunk(IFF_File *iff, Uint id, IFF_Chunk *chunk);
-void iff_dtor(IFF_File *iff);
 
 typedef struct {
     /* The encoding that was used to create this table. This is only used for
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index c55189f8a7..d1920bd072 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -462,8 +462,6 @@ bif erts_debug:lcnt_clear/0
 # New Bifs in R8.
 #
 
-bif code:get_chunk/2
-bif code:module_md5/1
 bif code:make_stub_module/3
 bif code:is_module_native/1
 
@@ -613,7 +611,6 @@ bif erlang:dt_append_vm_tag_data/1
 #
 # New in R16B.
 #
-bif erlang:prepare_loading/2
 bif erlang:finish_loading/1
 bif erlang:insert_element/3
 bif erlang:delete_element/2
@@ -769,3 +766,10 @@ bif erts_internal:abort_pending_connection/2
 #
 
 bif erts_internal:get_creation/0
+
+#
+# New in 24
+#
+bif erts_internal:prepare_loading/2
+bif erts_internal:beamfile_chunk/2
+bif erts_internal:beamfile_module_md5/1
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index 9bf538de29..9fe29483eb 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam
index f7650c1564..40ec184313 100644
--- a/erts/preloaded/ebin/erts_internal.beam
+++ b/erts/preloaded/ebin/erts_internal.beam
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 8cac9cd506..be1ffd0ea7 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -1552,9 +1552,29 @@ timestamp() ->
       Module :: module(),
       Code :: binary(),
       PreparedCode :: prepared_code(),
-      Reason :: bad_file.
-prepare_loading(_Module, _Code) ->
-    erlang:nif_error(undefined).
+      Reason :: badfile.
+prepare_loading(Module, <<"FOR1",_/bits>>=Code) ->
+    prepare_loading_1(Module, Code);
+prepare_loading(Module, Code0) ->
+    %% Corrupt header or compressed module, attempt to decompress it before
+    %% passing it to the loader and leave error signalling to the BIF.
+    Code = try zlib:gunzip(Code0) of
+               Decompressed -> Decompressed
+           catch
+               _:_ -> Code0
+           end,
+
+    prepare_loading_1(Module, Code).
+
+prepare_loading_1(Module, Code) ->
+    try erts_internal:prepare_loading(Module, Code) of
+        Res -> Res
+    catch
+        error:Reason ->
+            {'EXIT',{new_stacktrace,[{Mod,_,L,Loc}|Rest]}} =
+                (catch erlang:error(new_stacktrace, [Module,Code])),
+            erlang:raise(error, Reason, [{Mod,prepare_loading,L,Loc}|Rest])
+    end.
 
 %% pre_loaded/0
 -spec pre_loaded() -> [module()].
diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl
index aa3b6b753c..ec22ec918a 100644
--- a/erts/preloaded/src/erts_internal.erl
+++ b/erts/preloaded/src/erts_internal.erl
@@ -110,6 +110,8 @@
 
 -export([crasher/6]).
 
+-export([prepare_loading/2, beamfile_chunk/2, beamfile_module_md5/1]).
+
 %%
 %% Await result of send to port
 %%
@@ -922,3 +924,25 @@ crasher(Node,Mod,Fun,Args,Opts,Reason) ->
     error_logger:warning_msg("** Can not start ~w:~w,~w (~w) on ~w **~n",
 			     [Mod,Fun,Args,Opts,Node]),
     erlang:exit(Reason).
+
+%%
+%% Actual BIF for erlang:prepare_loading/2, which decompresses the module when
+%% necessary to save us from having to do it in C code.
+%%
+-spec prepare_loading(Module, Code) -> PreparedCode | {error, Reason} when
+      Module :: module(),
+      Code :: binary(),
+      PreparedCode :: erlang:prepared_code(),
+      Reason :: badfile.
+prepare_loading(_Module, _Code) ->
+    erlang:nif_error(undefined).
+
+-spec beamfile_chunk(Bin, Chunk) -> binary() | undefined when
+      Bin :: binary(),
+      Chunk :: string().
+beamfile_chunk(_Bin, _Chunk) ->
+    erlang:nif_error(undefined).
+
+-spec beamfile_module_md5(binary()) -> binary() | undefined.
+beamfile_module_md5(_Bin) ->
+    erlang:nif_error(undefined).
diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl
index 7b0ef8cf38..4461d54dc3 100644
--- a/lib/kernel/src/code.erl
+++ b/lib/kernel/src/code.erl
@@ -116,8 +116,22 @@
       Bin :: binary(),
       Chunk :: string().
 
-get_chunk(_, _) ->
-    erlang:nif_error(undef).
+get_chunk(<<"FOR1", _/bits>>=Beam, Chunk) ->
+    get_chunk_1(Beam, Chunk);
+get_chunk(Beam, Chunk) ->
+    %% Corrupt header or compressed module, decompress it before passing it to
+    %% the loader and let the BIF signal any errors.
+    get_chunk_1(try_decompress(Beam), Chunk).
+
+get_chunk_1(Beam, Chunk) ->
+    try
+        erts_internal:beamfile_chunk(Beam, Chunk)
+    catch
+        error:Reason ->
+            {'EXIT',{new_stacktrace,[{Mod,_,L,Loc}|Rest]}} =
+                (catch erlang:error(new_stacktrace, [Beam, Chunk])),
+            erlang:raise(error, Reason, [{Mod,get_chunk,L,Loc}|Rest])
+    end.
 
 -spec is_module_native(Module) -> true | false | undefined when
       Module :: module().
@@ -135,8 +149,29 @@ make_stub_module(_, _, _) ->
 
 -spec module_md5(binary()) -> binary() | undefined.
 
-module_md5(_) ->
-    erlang:nif_error(undef).
+module_md5(<<"FOR1", _/bits>>=Beam) ->
+    module_md5_1(Beam);
+module_md5(Beam) ->
+    %% Corrupt header or compressed module, decompress it before passing it to
+    %% the loader and let the BIF signal any errors.
+    module_md5_1(try_decompress(Beam)).
+
+module_md5_1(Beam) ->
+    try
+        erts_internal:beamfile_module_md5(Beam)
+    catch
+        error:Reason ->
+            {'EXIT',{new_stacktrace,[{Mod,_,L,Loc}|Rest]}} =
+                (catch erlang:error(new_stacktrace, [Beam])),
+            erlang:raise(error, Reason, [{Mod,module_md5,L,Loc}|Rest])
+    end.
+
+try_decompress(Bin0) ->
+    try zlib:gunzip(Bin0) of
+        Decompressed -> Decompressed
+    catch
+        _:_ -> Bin0
+    end.
 
 %%% End of BIFs
 
diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl
index df2e0ee64a..88c96223f9 100644
--- a/lib/kernel/test/code_SUITE.erl
+++ b/lib/kernel/test/code_SUITE.erl
@@ -842,7 +842,7 @@ analyse([], [This={M,F,A}|Path], Visited, ErrCnt0) ->
     OK = [erlang, os, prim_file, erl_prim_loader, init, ets,
 	  code_server, lists, lists_sort, unicode, binary, filename,
 	  gb_sets, gb_trees, hipe_unified_loader, hipe_bifs,
-	  erts_code_purger,
+	  erts_code_purger, erts_internal, code,
 	  prim_zip, zlib],
     ErrCnt1 =
 	case lists:member(M, OK) or erlang:is_builtin(M,F,A) of