Rework how a build file (Makefile, ...) is produced

The memory footprint of how we produced the Makefile was quite...
important, because we have all the processing in one perl snippet, and
generate the details of the build file by appending to the "magic"
variable $OUT.  The result is that this variable gets to hold the
majority of the build file text, and depending on memory reallocation
strategies for strings, the heap may hold multiple (possibly not just
a few) copies of this string, almost all of them "freed" but still
taking up space.  This has resulted in memory exhaustion.

We therefore change strategy, and generate the build file in two
phases, where the first phase generates the full template using small
perl snippets for each detail, and the second phase processes this
template.  This is much kinder to process memory.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15310)

1 files changed, 50 insertions, 29 deletions

diff --git a/configdata.pm.in b/configdata.pm.in
index 666d1f36d8..6c0d30baa0 100644
--- a/configdata.pm.in
+++ b/configdata.pm.in
@@ -53,6 +53,7 @@ unless (caller) {
     use Getopt::Long;
     use File::Spec::Functions;
     use File::Basename;
+    use File::Copy;
     use Pod::Usage;
 
     use lib '{- sourcedir('util', 'perl') -}';
@@ -62,6 +63,39 @@ unless (caller) {
 
     if (scalar @ARGV == 0) {
         # With no arguments, re-create the build file
+        # We do that in two steps, where the first step emits perl
+        # snipets.
+
+        my $buildfile = $target{build_file};
+        my $buildfile_template = "$buildfile.in";
+        my @autowarntext = (
+            'WARNING: do not edit!',
+            "Generated by configdata.pm from "
+            .join(", ", @{$config{build_file_templates}}),
+            "via $buildfile_template"
+        );
+        my %gendata = (
+            config => \%config,
+            target => \%target,
+            disabled => \%disabled,
+            withargs => \%withargs,
+            unified_info => \%unified_info,
+            autowarntext => \@autowarntext,
+            );
+
+        use lib '.';
+        use lib '{- sourcedir('Configurations') -}';
+        use gentemplate;
+
+        print 'Creating ',$buildfile_template,"\n";
+        open my $buildfile_template_fh, ">$buildfile_template"
+            or die "Trying to create $buildfile_template: $!";
+        foreach (@{$config{build_file_templates}}) {
+            copy($_, $buildfile_template_fh)
+                or die "Trying to copy $_ into $buildfile_template: $!";
+        }
+        gentemplate(output => $buildfile_template_fh, %gendata);
+        close $buildfile_template_fh;
 
         use OpenSSL::Template;
 
@@ -73,36 +107,23 @@ use lib '{- $config{builddir} -}';
 use platform;
 _____
 
-        my @autowarntext = (
-            'WARNING: do not edit!',
-            "Generated by configdata.pm from "
-            .join(", ", @{$config{build_file_templates}})
-        );
-
-        print 'Creating ',$target{build_file},"\n";
-        open BUILDFILE, ">$target{build_file}.new"
-            or die "Trying to create $target{build_file}.new: $!";
-        foreach (@{$config{build_file_templates}}) {
-            my $tmpl = OpenSSL::Template->new(TYPE => 'FILE',
-                                              SOURCE => $_);
-            $tmpl->fill_in(FILENAME => $_,
-                           OUTPUT => \*BUILDFILE,
-                           HASH => { config => \%config,
-                                     target => \%target,
-                                     disabled => \%disabled,
-                                     withargs => \%withargs,
-                                     unified_info => \%unified_info,
-                                     autowarntext => \@autowarntext },
-                           PREPEND => $prepend,
-                           # To ensure that global variables and functions
-                           # defined in one template stick around for the
-                           # next, making them combinable
-                           PACKAGE => 'OpenSSL::safe')
-                or die $Text::Template::ERROR;
-        }
+        print 'Creating ',$buildfile,"\n";
+        open BUILDFILE, ">$buildfile.new"
+            or die "Trying to create $buildfile.new: $!";
+        my $tmpl = OpenSSL::Template->new(TYPE => 'FILE',
+                                          SOURCE => $buildfile_template);
+        $tmpl->fill_in(FILENAME => $_,
+                       OUTPUT => \*BUILDFILE,
+                       HASH => \%gendata,
+                       PREPEND => $prepend,
+                       # To ensure that global variables and functions
+                       # defined in one template stick around for the
+                       # next, making them combinable
+                       PACKAGE => 'OpenSSL::safe')
+            or die $Text::Template::ERROR;
         close BUILDFILE;
-        rename("$target{build_file}.new", $target{build_file})
-            or die "Trying to rename $target{build_file}.new to $target{build_file}: $!";
+        rename("$buildfile.new", $buildfile)
+            or die "Trying to rename $buildfile.new to $buildfile: $!";
 
         exit(0);
     }