Merge branch 'deterministic-build' of https://github.com/TD5/otp into maint

* 'deterministic-build' of https://github.com/TD5/otp:
  make: Allow OTP to be built deterministically
  compiler: Make test_lib robust to +deterministic
  compiler: Make compiler forward +determinsitic flag to epp
  compiler: Make yecc respect +deterministic
  compiler: Make leex respect +deterministic
  compiler: Make asn1ct_gen respect +deterministic
  compiler: Make EPP respect +deterministic

OTP-18165

7 files changed, 160 insertions, 29 deletions

diff --git a/lib/parsetools/doc/src/leex.xml b/lib/parsetools/doc/src/leex.xml
index 357ed86f44..d802e46b59 100644
--- a/lib/parsetools/doc/src/leex.xml
+++ b/lib/parsetools/doc/src/leex.xml
@@ -110,6 +110,11 @@
             <item>
               <p>Causes warnings to be treated as errors.</p>
             </item>
+            <tag><c>{deterministic, boolean()}</c></tag>
+            <item>
+              <p>Causes generated -file() attributes to only include
+                the basename of the file path.</p>
+            </item>
           </taglist>
         <p>Any of the Boolean options can be set to <c>true</c> by 
           stating the name of the option. For example, <c>verbose</c>
diff --git a/lib/parsetools/doc/src/yecc.xml b/lib/parsetools/doc/src/yecc.xml
index 218cec5330..4d639d1f21 100644
--- a/lib/parsetools/doc/src/yecc.xml
+++ b/lib/parsetools/doc/src/yecc.xml
@@ -134,6 +134,11 @@
             is <c>column</c>, the location includes a line number and
             a column number. Default is <c>column</c>.
           </item>
+          <tag><c>{deterministic, boolean()}</c></tag>
+          <item>
+            <p>Causes generated -file() attributes to only include the
+              basename of the file path.</p>
+          </item>
         </taglist>
         <p>Any of the Boolean options can be set to <c>true</c> by 
           stating the name of the option. For example, <c>verbose</c>
diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl
index 15803b3b82..b764678516 100644
--- a/lib/parsetools/src/leex.erl
+++ b/lib/parsetools/src/leex.erl
@@ -78,9 +78,10 @@ compile(Input0, Output0,
     Output = assure_extension(shorten_filename(Output0), ".erl"),
     Includefile = lists:sublist(Includes, 1),
     Werror = proplists:get_bool(warnings_as_errors, Specific),
+    Deterministic = proplists:get_bool(deterministic, Specific),
     Opts = [{scannerfile,Output},{includefile,Includefile},{verbose,Verbose},
             {report_errors,true},{report_warnings,WarnLevel > 0},
-	    {warnings_as_errors, Werror}],
+	    {warnings_as_errors, Werror}, {deterministic, Deterministic}],
     case file(Input, Opts) of
         {ok, _} ->
             ok;
@@ -117,6 +118,7 @@ file(File) -> file(File, []).
               | {'scannerfile', Scannerfile :: file:filename()}
               | {'verbose', boolean()}
               | {'warnings_as_errors', boolean()}
+              | {'deterministic', boolean()}
               | 'dfa_graph'
               | 'report_errors' | 'report_warnings' | 'report'
               | 'return_errors' | 'return_warnings' | 'return'
@@ -287,7 +289,7 @@ check_options(_Options, _, _L) ->
 all_options() ->
     [dfa_graph,includefile,report_errors,report_warnings,
      return_errors,return_warnings,scannerfile,verbose,
-     warnings_as_errors].
+     warnings_as_errors, deterministic].
 
 default_option(dfa_graph) -> false;
 default_option(includefile) -> [];
@@ -297,7 +299,8 @@ default_option(return_errors) -> false;
 default_option(return_warnings) -> false;
 default_option(scannerfile) -> [];
 default_option(verbose) -> false;
-default_option(warnings_as_errors) -> false.
+default_option(warnings_as_errors) -> false;
+default_option(deterministic) -> false.
 
 atom_option(dfa_graph) -> {dfa_graph,true};
 atom_option(report_errors) -> {report_errors,true};
@@ -306,6 +309,7 @@ atom_option(warnings_as_errors) -> {warnings_as_errors,true};
 atom_option(return_errors) -> {return_errors,true};
 atom_option(verbose) -> {verbose,true};
 atom_option(return_warnings) -> {return_warnings,true};
+atom_option(deterministic) -> {deterministic,true};
 atom_option(Key) -> Key.
 
 is_filename(T) ->
@@ -1362,7 +1366,8 @@ out_file(St0, DFA, DF, Actions, Code) ->
                         set_encoding(St0, Ofile),
                         try 
                             output_encoding_comment(Ofile, St0),
-                            output_file_directive(Ofile, St0#leex.ifile, 0),
+                            Deterministic = proplists:get_bool(deterministic, St0#leex.opts),
+                            output_file_directive(Ofile, St0#leex.ifile, Deterministic, 0),
                             out_file(Ifile, Ofile, St0, DFA, DF, Actions,
                                      Code, 1),
                             verbose_print(St0, "ok~n", []),
@@ -1400,15 +1405,18 @@ inc_file_name(Filename) ->
 %%  characters.
 
 out_file(Ifile, Ofile, St, DFA, DF, Actions, Code, L) ->
+    Deterministic = proplists:get_bool(deterministic, St#leex.opts),
     case io:get_line(Ifile, leex) of
-        eof -> output_file_directive(Ofile, St#leex.ifile, L);
-        {error, _} -> add_error(St#leex.ifile, {L, leex, cannot_parse}, St);
+        eof ->
+            output_file_directive(Ofile, St#leex.ifile, Deterministic, L);
+        {error, _} ->
+            add_error(St#leex.ifile, {L, leex, cannot_parse}, St);
         Line ->
             case string:slice(Line, 0, 5) of
                 "##mod" -> out_module(Ofile, St);
                 "##cod" -> out_erlang_code(Ofile, St, Code, L);
                 "##dfa" -> out_dfa(Ofile, St, DFA, Code, DF, L);
-                "##act" -> out_actions(Ofile, St#leex.xfile, Actions);
+                "##act" -> out_actions(Ofile, St#leex.xfile, Deterministic, Actions);
                 _ -> io:put_chars(Ofile, Line)
             end,
             out_file(Ifile, Ofile, St, DFA, DF, Actions, Code, L+1)
@@ -1419,7 +1427,8 @@ out_module(File, St) ->
 
 out_erlang_code(File, St, Code, L) ->
     {CodeL,CodePos,_NCodeLines} = Code,
-    output_file_directive(File, St#leex.xfile, CodeL),
+    Deterministic = proplists:get_bool(deterministic, St#leex.opts),
+    output_file_directive(File, St#leex.xfile, Deterministic, CodeL),
     {ok,Xfile} = file:open(St#leex.xfile, [read]),
     try
         set_encoding(St, Xfile),
@@ -1429,7 +1438,7 @@ out_erlang_code(File, St, Code, L) ->
         ok = file:close(Xfile)
     end,
     io:nl(File),
-    output_file_directive(File, St#leex.ifile, L).
+    output_file_directive(File, St#leex.ifile, Deterministic, L).
 
 file_copy(From, To) ->
     case io:get_line(From, leex) of
@@ -1441,8 +1450,9 @@ file_copy(From, To) ->
 
 out_dfa(File, St, DFA, Code, DF, L) ->
     {_CodeL,_CodePos,NCodeLines} = Code,
+    Deterministic = proplists:get_bool(deterministic, St#leex.opts),
     %% Three file attributes before this one...
-    output_file_directive(File, St#leex.efile, L+(NCodeLines-1)+3),
+    output_file_directive(File, St#leex.efile, Deterministic, L+(NCodeLines-1)+3),
     io:fwrite(File, "yystate() -> ~w.~n~n", [DF]),
     foreach(fun (S) -> out_trans(File, S) end, DFA),
     io:fwrite(File, "yystate(S, Ics, Line, Tlen, Action, Alen) ->~n", []),
@@ -1565,14 +1575,14 @@ pack_trans([Tr|Trs], Pt) ->                % The default uninteresting case
     pack_trans(Trs, Pt ++ [Tr]);
 pack_trans([], Pt) -> Pt.
 
-%% out_actions(File, XrlFile, ActionList) -> ok.
+%% out_actions(File, XrlFile, Deterministic, ActionList) -> ok.
 %% Write out the action table.
 
-out_actions(File, XrlFile, As) ->
+out_actions(File, XrlFile, Deterministic, As) ->
     As1 = prep_out_actions(As),
     foreach(fun (A) -> out_action(File, A) end, As1),
     io:fwrite(File, "yyaction(_, _, _, _) -> error.~n", []),
-    foreach(fun (A) -> out_action_code(File, XrlFile, A) end, As1).
+    foreach(fun (A) -> out_action_code(File, XrlFile, Deterministic, A) end, As1).
 
 prep_out_actions(As) ->
     map(fun ({A,empty_action}) ->
@@ -1603,14 +1613,14 @@ out_action(File, {A,_Code,Vars,Name,_Args,ArgsChars}) ->
     end,
     io:fwrite(File, "    ~s(~s);~n", [Name, ArgsChars]).
 
-out_action_code(_File, _XrlFile, {_A,empty_action}) ->
+out_action_code(_File, _XrlFile, _Deterministic, {_A,empty_action}) ->
     ok;
-out_action_code(File, XrlFile, {_A,Code,_Vars,Name,Args,ArgsChars}) ->
+out_action_code(File, XrlFile, Deterministic, {_A,Code,_Vars,Name,Args,ArgsChars}) ->
     %% Should set the file to the .erl file, but instead assumes that
     %% ?LEEXINC is syntactically correct.
     io:fwrite(File, "\n-compile({inline,~w/~w}).\n", [Name, length(Args)]),
     L = erl_scan:line(hd(Code)),
-    output_file_directive(File, XrlFile, L-2),
+    output_file_directive(File, XrlFile, Deterministic, L-2),
     io:fwrite(File, "~s(~s) ->~n", [Name, ArgsChars]),
     io:fwrite(File, "    ~ts\n", [pp_tokens(Code, L, File)]).
 
@@ -1710,12 +1720,18 @@ output_encoding_comment(_File, #leex{encoding = none}) ->
 output_encoding_comment(File, #leex{encoding = Encoding}) ->
     io:fwrite(File, <<"%% ~s\n">>, [epp:encoding_to_string(Encoding)]).
 
-output_file_directive(File, Filename, Line) ->
+output_file_directive(File, Filename, Deterministic, Line) ->
     io:fwrite(File, <<"-file(~ts, ~w).\n">>,
-              [format_filename(Filename, File), Line]).
-
-format_filename(Filename0, File) ->
-    Filename = filename:flatten(Filename0),
+              [format_filename(Filename, File, Deterministic), Line]).
+
+format_filename(Filename0, File, Deterministic) ->
+    Filename =
+        case Deterministic of
+            true ->
+                filename:basename(filename:flatten(Filename0));
+            false ->
+                filename:flatten(Filename0)
+        end,
     case enc(File) of
         unicode -> io_lib:write_string(Filename);
         latin1  -> io_lib:write_string_as_latin1(Filename)
diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl
index 5b2a9efe4d..7833c6a120 100644
--- a/lib/parsetools/src/yecc.erl
+++ b/lib/parsetools/src/yecc.erl
@@ -141,9 +141,10 @@ compile(Input0, Output0,
     Output = shorten_filename(Output0),
     Includefile = lists:sublist(Includes, 1),
     Werror = proplists:get_bool(warnings_as_errors, Specific),
+    Deterministic = proplists:get_bool(deterministic, Specific),
     Opts = [{parserfile,Output}, {includefile,Includefile}, {verbose,Verbose},
             {report_errors, true}, {report_warnings, WarnLevel > 0},
-	    {warnings_as_errors, Werror}],
+	    {warnings_as_errors, Werror}, {deterministic, Deterministic}],
     case file(Input, Opts) of
         {ok, _OutFile} ->
             ok;
@@ -265,6 +266,7 @@ file(GrammarFile) ->
               | {'parserfile', Parserfile :: file:filename()}
               | {'verbose', boolean()}
               | {'warnings_as_errors', boolean()}
+              | {'deterministic', boolean()}
               | 'report_errors' | 'report_warnings' | 'report'
               | 'return_errors' | 'return_warnings' | 'return'
               | 'verbose' | 'warnings_as_errors'.
@@ -407,7 +409,7 @@ check_options(_Options, _, _L) ->
 all_options() ->
     [error_location, file_attributes, includefile, parserfile,
      report_errors, report_warnings, return_errors, return_warnings,
-     time, verbose, warnings_as_errors].
+     time, verbose, warnings_as_errors, deterministic].
 
 default_option(error_location) -> column;
 default_option(file_attributes) -> true;
@@ -419,7 +421,8 @@ default_option(return_errors) -> false;
 default_option(return_warnings) -> false;
 default_option(time) -> false;
 default_option(verbose) -> false;
-default_option(warnings_as_errors) -> false.
+default_option(warnings_as_errors) -> false;
+default_option(deterministic) -> false.
 
 atom_option(file_attributes) -> {file_attributes, true};
 atom_option(report_errors) -> {report_errors, true};
@@ -429,6 +432,7 @@ atom_option(return_warnings) -> {return_warnings, true};
 atom_option(time) -> {time, true};
 atom_option(verbose) -> {verbose, true};
 atom_option(warnings_as_errors) -> {warnings_as_errors, true};
+atom_option(deterministic) -> {deterministic, true};
 atom_option(Key) -> Key.
 
 is_filename(T) ->
@@ -2695,7 +2699,12 @@ nl(#yecc{outport = Outport, line = Line}=St) ->
     St#yecc{line = Line + 1}.
 
 format_filename(Filename0, St) ->
-    Filename = filename:flatten(Filename0),
+    Deterministic = proplists:get_bool(deterministic, St#yecc.options),
+    Filename =
+      case Deterministic of
+        true -> filename:basename(filename:flatten(Filename0));
+        false -> filename:flatten(Filename0)
+      end,
     case lists:keyfind(encoding, 1, io:getopts(St#yecc.outport)) of
         {encoding, unicode} -> io_lib:write_string(Filename);
         _ ->                   io_lib:write_string_as_latin1(Filename)
diff --git a/lib/parsetools/test/Makefile b/lib/parsetools/test/Makefile
index d494953f1e..23cb9709e1 100644
--- a/lib/parsetools/test/Makefile
+++ b/lib/parsetools/test/Makefile
@@ -42,6 +42,7 @@ RELSYSDIR = $(RELEASE_PATH)/parsetools_test
 # ----------------------------------------------------
 ERL_MAKE_FLAGS += 
 ERL_COMPILE_FLAGS +=
+ERL_COMPILE_FLAGS := $(filter-out +deterministic,$(ERL_COMPILE_FLAGS))
 
 EBIN = .
 
diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl
index 09a6e026bd..bef048dc82 100644
--- a/lib/parsetools/test/leex_SUITE.erl
+++ b/lib/parsetools/test/leex_SUITE.erl
@@ -22,6 +22,7 @@
 %-define(debug, true).
 
 -include_lib("stdlib/include/erl_compile.hrl").
+-include_lib("stdlib/include/assert.hrl").
 -include_lib("kernel/include/file.hrl").
 
 -ifdef(debug).
@@ -39,7 +40,7 @@
 	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([
-	 file/1, compile/1, syntax/1,
+	 file/1, compile/1, syntax/1, deterministic/1,
 	 
 	 pt/1, man/1, ex/1, ex2/1, not_yet/1,
 	 line_wrap/1,
@@ -64,7 +65,7 @@ all() ->
     [{group, checks}, {group, examples}, {group, tickets}, {group, bugs}].
 
 groups() -> 
-    [{checks, [], [file, compile, syntax]},
+    [{checks, [], [file, compile, syntax, deterministic]},
      {examples, [], [pt, man, ex, ex2, not_yet, unicode]},
      {tickets, [], [otp_10302, otp_11286, otp_13916, otp_14285, otp_17023,
                     compiler_warnings]},
@@ -368,6 +369,42 @@ syntax(Config) when is_list(Config) ->
         leex:file(Filename, Ret),
     ok.
 
+deterministic(doc) ->
+    "Check leex respects the +deterministic flag.";
+deterministic(suite) -> [];
+deterministic(Config) when is_list(Config) ->
+    Dir = ?privdir,
+    Filename = filename:join(Dir, "file.xrl"),
+    Scannerfile = filename:join(Dir, "file.erl"),
+    Mini = <<"Definitions.\n"
+             "D  = [0-9]\n"
+             "Rules.\n"
+             "{L}+  : {token,{word,TokenLine,TokenChars}}.\n"
+             "Erlang code.\n">>,
+    ok = file:write_file(Filename, Mini),
+
+    %% Generated leex scanners include the leexinc.hrl header file by default,
+    %% so we'll get a -file attribute corresponding to that include. In
+    %% deterministic mode, that include should only use the basename,
+    %% "leexinc.hrl", but otherwise, it should contain the full path.
+
+    %% Matches when OTP is not installed (e.g. /lib/parsetools/include/leexinc.hrl)
+    %% and when it is (e.g. /lib/parsetools-2.3.2/include/leexinc.hrl)
+    AbsolutePathSuffix = ".*/lib/parsetools.*/include/leexinc\.hrl",
+
+    ok = leex:compile(Filename, Scannerfile, #options{specific=[deterministic]}),
+    {ok, FormsDet} = epp:parse_file(Scannerfile,[]),
+    ?assertMatch(false, search_for_file_attr(AbsolutePathSuffix, FormsDet)),
+    ?assertMatch({value, _}, search_for_file_attr("leexinc\.hrl", FormsDet)),
+    file:delete(Scannerfile),
+
+    ok = leex:compile(Filename, Scannerfile, #options{}),
+    {ok, Forms} = epp:parse_file(Scannerfile,[]),
+    ?assertMatch({value, _}, search_for_file_attr(AbsolutePathSuffix, Forms)),
+    file:delete(Scannerfile),
+
+    file:delete(Filename),
+    ok.
 
 pt(doc) ->
     "Pushing back characters.";
@@ -1272,3 +1309,13 @@ extract(File, {error, Es, Ws}) ->
     {errors, extract(File, Es), extract(File, Ws)};    
 extract(File, Ts) ->
     lists:append([T || {F, T} <- Ts,  F =:= File]).
+
+search_for_file_attr(PartialFilePathRegex, Forms) ->
+    lists:search(fun
+                   ({attribute, _, file, {FileAttr, _}}) ->
+                      case re:run(FileAttr, PartialFilePathRegex) of
+                        nomatch -> false;
+                        _ -> true
+                      end;
+                   (_) -> false end,
+                 Forms).
diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl
index c0f03edd9e..e76b98f0f5 100644
--- a/lib/parsetools/test/yecc_SUITE.erl
+++ b/lib/parsetools/test/yecc_SUITE.erl
@@ -22,6 +22,7 @@
 %-define(debug, true).
 
 -include_lib("stdlib/include/erl_compile.hrl").
+-include_lib("stdlib/include/assert.hrl").
 
 -ifdef(debug).
 -define(config(X,Y), foo).
@@ -40,7 +41,7 @@
 -export([app_test/1,
 	 
 	 file/1, syntax/1, compile/1, rules/1, expect/1,
-	 conflicts/1,
+	 conflicts/1, deterministic/1,
 	 
 	 empty/1, prec/1, yeccpre/1, lalr/1, old_yecc/1, 
 	 other_examples/1,
@@ -70,7 +71,7 @@ all() ->
 
 groups() -> 
     [{checks, [],
-      [file, syntax, compile, rules, expect, conflicts]},
+      [file, syntax, compile, rules, expect, conflicts, deterministic]},
      {examples, [],
       [empty, prec, yeccpre, lalr, old_yecc, other_examples]},
      {bugs, [],
@@ -926,6 +927,43 @@ conflicts(Config) when is_list(Config) ->
     file:delete(Filename),
     ok.
 
+deterministic(doc) ->
+    "Check yecc respects the +deterministic flag.";
+deterministic(suite) -> [];
+deterministic(Config) when is_list(Config) ->
+    Dir = ?privdir,
+    Filename = filename:join(Dir, "file.yrl"),
+    Parserfile = filename:join(Dir, "file.erl"),
+    ok = file:write_file(Filename,
+                               <<"Nonterminals nt.
+                                  Terminals t.
+                                  Rootsymbol nt.
+                                  nt -> t.">>),
+
+    %% Generated yecc parsers need to include the yeccpre.hrl
+    %% header file, so we'll get a -file attribute corresponding
+    %% to that include. In deterministic mode, that include should
+    %% only use the basename, "yeccpre.hrl", but otherwise, it should
+    %% contain the full path.
+
+    %% Matches when OTP is not installed (e.g. /lib/parsetools/include/yeccpre.hrl)
+    %% and when it is (e.g. /lib/parsetools-2.3.2/include/yeccpre.hrl)
+    AbsolutePathSuffix = "/lib/parsetools.*/include/yeccpre\.hrl",
+
+    ok = yecc:compile(Filename, Parserfile, #options{specific=[deterministic]}),
+    {ok, FormsDet} = epp:parse_file(Parserfile,[]),
+    ?assertMatch(false, search_for_file_attr(AbsolutePathSuffix, FormsDet)),
+    ?assertMatch({value, _}, search_for_file_attr("yeccpre\.hrl", FormsDet)),
+    file:delete(Parserfile),
+
+    ok = yecc:compile(Filename, Parserfile, #options{}),
+    {ok, Forms} = epp:parse_file(Parserfile,[]),
+    ?assertMatch({value, _}, search_for_file_attr(AbsolutePathSuffix, Forms)),
+    file:delete(Parserfile),
+
+    file:delete(Filename),
+    ok.
+
 empty(doc) ->
     "'$empty'.";
 empty(suite) -> [];
@@ -2284,3 +2322,13 @@ process_list() ->
 
 safe_second_element({_,Info}) -> Info;
 safe_second_element(Other) -> Other.
+
+search_for_file_attr(PartialFilePathRegex, Forms) ->
+    lists:search(fun
+                   ({attribute, _, file, {FileAttr, _}}) ->
+                      case re:run(FileAttr, PartialFilePathRegex) of
+                        nomatch -> false;
+                        _ -> true
+                      end;
+                   (_) -> false end,
+                 Forms).