util_precompile: add ability to reuse previously created blobs

Most strings in the Cr50 image are not going to change any time soon,
the most likely change is the addition of new strings.

With this in mind, it makes sense to keep the string dictionary around
and just keep updating it when building a new image, maybe adding new
strings, or most often just reusing the ones already in the blob.

This patch tries reading the blob before processing the inputs, and if
the blob is read and unzipped/unpickled successfully, its contents are
used to seed the string dictionary before the inputs are processed.

With this approach 100% compatibility between the Cr50 image and the
blob is not required, most of the strings would be interpreted
properly even with older blobs.

A test case is being added to verify this behavior.

BUG=b:149964350
TEST=./util/test_util_precompile.py

Signed-off-by: Vadim Bendebury <vbendeb@chromium.org>
Change-Id: I761d6b043dfc85b54352aadaa66d24d4027e95f8
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/2119144

2 files changed, 69 insertions, 0 deletions

diff --git a/util/test_util_precompile.py b/util/test_util_precompile.py
index f27f6c4a2f..210130c12c 100755
--- a/util/test_util_precompile.py
+++ b/util/test_util_precompile.py
@@ -5,7 +5,9 @@
 # found in the LICENSE file.
 'Unit tests for util_precompile.py'
 
+import os
 import pickle
+import tempfile
 import unittest
 import zlib
 
@@ -137,6 +139,50 @@ class TestUtilPrecompile(unittest.TestCase):
             else:
                 self.fail('did not find "%s" in the dictionary')
 
+    def test_incremental_blob(self):
+        """Verify that string blob is properly extended.
+
+        When invoked with an existing blob, util_precompile.py is supposed to
+        re-use existing strings and only add new ones.
+
+        Create a test file with a set of strings, generate the blob, then
+        create another test file, with an extra string inserted in the
+        beginning and verify, generate the blob again, and verify that the
+        resulting blob has the strings at expected indices.
+        """
+        first_string_set = ('format string #1',
+                            'format string #2',
+                            'format_strint #3')
+
+        second_string_set = ('format string #4',) + first_string_set
+        with tempfile.TemporaryDirectory(prefix='test_uc') as td:
+            source_code = os.path.join(td, 'src.E')
+            lock_file = os.path.join(td, 'lock')
+            blob = os.path.join(td, 'blob')
+            util_precompile.FMT_DICT = {}
+
+            def prepare_source_code(file_name, strings):
+                'Generate test .E file given a list of text strings'
+                with open(file_name, 'w') as sf:
+                    for string in strings:
+                        sf.write(' cprintf(CHAN, "%s");\n' % string)
+
+            prepare_source_code(source_code, first_string_set)
+            util_precompile.main(['_', '-o', blob, '-l',
+                                  lock_file, source_code])
+            prepare_source_code(source_code, second_string_set)
+            util_precompile.FMT_DICT = {}
+            util_precompile.main(['_', '-o', blob, '-l',
+                                  lock_file, source_code])
+
+            # Verify that strings in the blob are at the expected indices.
+            # The first set strings should have lower indices.
+            for i, string in enumerate(first_string_set):
+                self.assertEqual(util_precompile.FMT_DICT.get(string, None), i)
+            # The second set strings should have higher indices.
+            string = second_string_set[0]
+            i = len(second_string_set) - 1
+            self.assertEqual(util_precompile.FMT_DICT.get(string, None), i)
 
     def test_drop_escapes(self):
         'Verify proper conversion of escape characters'
@@ -147,5 +193,6 @@ class TestUtilPrecompile(unittest.TestCase):
         for i, o in insouts:
             self.assertEqual(o, util_precompile.drop_escapes(i))
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/util/util_precompile.py b/util/util_precompile.py
index c58b811913..bc8ad0320f 100755
--- a/util/util_precompile.py
+++ b/util/util_precompile.py
@@ -521,6 +521,27 @@ def generate_blob():
     print('dump size %d, compressed size %d' % (len(dump), len(zipped)))
     return zipped
 
+def seed_blob(outfile):
+    """Read string data from a previously saved blob
+
+    This function is invoked only if the blob file exists.
+    """
+    global FMT_DICT
+    if outfile.endswith('Ep'):
+        print('invoked with', ' '.join(sys.argv))
+        assert False
+    with open(outfile, 'rb') as blob:
+        try:
+            zipped = blob.read()
+            pickled = zlib.decompress(zipped)
+            dump = pickle.loads(pickled)
+        except (zlib.error, pickle.UnpicklingError):
+            print('%s does not seem to be a proper blob, ignored' % outfile)
+            return
+        strings = dump.split('\0')
+        for i in range(len(strings)):
+            FMT_DICT[strings[i]] = i
+
 def main(argv):
     """Main function.
 
@@ -546,6 +567,7 @@ def main(argv):
             else:
                 # Output file is newer than all inputs.
                 return
+            seed_blob(flags.output)
 
         for e_file in files[1:]:
             preobj_process(e_file, flags.ext)