script for processing preprocessor output

This script is customized to be used in Chrome OS EC codebase, where
functions cprintf(), cprints() and cputs()are used to generate console
output.

The operation of the script is extensively described in the docstring
of the file, a unit test is provided to verify basic functionality.

BUG=b:149964350
TEST=./util/test_util_precompile.py succeeds with the rest of the
     patches applied built a Cr50 image, verified that using a
     verified version of Acroterm one can communicate with Cr50
     console the same way it works today over regular UART.

Change-Id: I2c92088a8b08dcf7fdb606a01fbc94ab0d2df6e6
Signed-off-by: Vadim Bendebury <vbendeb@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/2106863
Reviewed-by: Randall Spangler <rspangler@chromium.org>

2 files changed, 715 insertions, 0 deletions

diff --git a/util/test_util_precompile.py b/util/test_util_precompile.py
new file mode 100755
index 0000000000..f27f6c4a2f
--- /dev/null
+++ b/util/test_util_precompile.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-"
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+'Unit tests for util_precompile.py'
+
+import pickle
+import unittest
+import zlib
+
+import util_precompile
+
+class TestUtilPrecompile(unittest.TestCase):
+    'Test class for testing various util_precompile.py functions'
+
+    def test_generate_cmsg_line(self):
+        'Test generating of cmsgX() lines from format messages/arguments'
+        # A tuple of tuples of typical format strings, following the matching
+        # number of parameters, the last element is the expected parameter
+        # description mask, describing the variety of format specifications in
+        # the format string.
+        inputs = (('"%s\n', 'errmsgs[rv]', 3),
+                  ('%d', 10, 1),
+                  ('"no format"', ),
+                  ('[%pT CCD state:', '((void *)0)', 7),
+                  ('%08x: DIO%c%-2d  %2d %3s%3s%3s%4s ',
+                   1, 2, 3, 4, 5, 6, 6, 7, 858984721))
+        string_indices = []
+        for inp in inputs:
+            fmt = inp[0]
+            if len(inp) > 1:
+                params = inp[1:-1]
+                mask = inp[-1]
+            else:
+                params = []
+                mask = None
+            blocks = fmt.split('%')[1:]
+            line = util_precompile.generate_cmsg_line(
+                fmt, params, fmt.split('%')[1:], 'chan', 'func')
+            exp_start = 'cmsg%d(chan, ' % len(blocks)
+            if params:
+                args = '%s' % (', '.join(['(uintptr_t)(%s)' % x
+                                          for x in params]))
+                exp_end = ', %d, %s);\n' % (mask, args)
+            else:
+                exp_end = ');\n'
+            try:
+                self.assertTrue(line.startswith(exp_start))
+                self.assertTrue(line.endswith(exp_end))
+            except AssertionError:
+                print('line: %s\nexp_start: %s\nexp_end: %s' % (
+                    line, exp_start, exp_end))
+                raise
+            line = line.replace(exp_start, '', 1)
+            line = line.replace(exp_end, '', 1)
+            string_indices.append(int(line))
+
+        # Verify the contents of the generated blob
+        zipped = util_precompile.generate_blob()
+        dump = zlib.decompress(zipped)
+        strings = pickle.loads(dump).split('\0')
+        for inp, index in zip(inputs, string_indices):
+            string = inp[0]
+            self.assertTrue(strings[index] == string)
+
+    def test_tokenize(self):
+        'Verify tokenize() function ability to parse vararg string'
+        in_tokens = (
+            '"simple string"',
+            '"another string, with a comma"',
+            '"string split" " in two"',
+            '"string with \\"escaped\\" double quotes"',
+            '(&(const struct hex_buffer_params)'
+            '{ .buffer = (ec_efs_ctx.hash), .size = (32) })')
+        out_tokens = util_precompile.tokenize(', '.join(in_tokens))
+        for in_t, out_t in zip(in_tokens, out_tokens):
+            self.assertEqual(in_t, out_t)
+
+    def test_line_processor(self):
+        'Test line processor class ability to consolidate preprocessor lines'
+        # Reset the string dictionary.
+        util_precompile.FMT_DICT = {}
+        in_out_tuples = (
+            (
+                """ cprintf(CC_COMMAND, "Last attempt returned " "%d\\n", rv)
+
+                ;""",
+
+                ' cmsg1(CC_COMMAND, 0, 1, (uintptr_t)(rv));\n'
+            ), (
+                ' cprintf(CC_COMMAND, "ec_hash_secdata    : %ph\\n", '
+                '(&(const struct hex_buffer_params)'
+                '{ .buffer = (ec_efs_ctx.hash), .size = (32) }));',
+                ' cmsg1(CC_COMMAND, 1, 5, (uintptr_t)((&(const struct '
+                'hex_buffer_params)'
+                '{ .buffer = (ec_efs_ctx.hash), .size = (32) })));\n'
+            ), (
+                'struct ec_params_get_cmd_versions {',
+                'struct ec_params_get_cmd_versions {',
+            ), (
+                '  cprints(CC_CCD, "CCD test lab mode %sbled", v '
+                '? "ena" : "disa");',
+                ' cmsg1(CC_CCD, 2, 3, (uintptr_t)(v ? "ena" : "disa"));\n'
+            ), (
+                '  cprintf(CC_COMMAND, "%s: deleting var failed!\\n", '
+                '__func__);',
+                ' cmsg1(CC_COMMAND, 3, 6, (uintptr_t)4);\n'
+            ), (
+                ' return  cprintf(CC_COMMAND, "%s: done!\\n", __func__);',
+                ' return cmsg1(CC_COMMAND, 5, 6, (uintptr_t)4);\n'
+            )
+        )
+        line_processor = util_precompile.LineProcessor()
+        for inp, outp in in_out_tuples:
+            result = ''
+            for line in inp.splitlines():
+                section = line_processor.process_preprocessor_line(line)
+                if section:
+                    result += section
+            self.assertEqual(result, outp)
+            try:
+                string_index = int(outp.split(',')[1])
+            except IndexError:
+                # This is the line without a print statement.
+                continue
+            fmt = inp.split(',')[1].strip()
+            for key, value in util_precompile.FMT_DICT.items():
+                if value == string_index:
+                    fmt = fmt.replace('" "', '')
+                    fmt = fmt.replace('\\n', '\n')
+                    fmt = fmt.strip('"')
+                    if 'cprints' in inp:
+                        fmt = '[^T' + fmt
+                    self.assertEqual(fmt, key)
+                    break
+            else:
+                self.fail('did not find "%s" in the dictionary')
+
+
+    def test_drop_escapes(self):
+        'Verify proper conversion of escape characters'
+        insouts = (('\\a\\b\\f\\n\\r\\t\\v\'"\\\\',
+                    '\a\b\f\n\r\t\v\'"\\'),
+                   ('line \\x1a with two hex escapes \\x1b\\n',
+                    'line \x1a with two hex escapes \x1b\n'))
+        for i, o in insouts:
+            self.assertEqual(o, util_precompile.drop_escapes(i))
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/util/util_precompile.py b/util/util_precompile.py
new file mode 100755
index 0000000000..c58b811913
--- /dev/null
+++ b/util/util_precompile.py
@@ -0,0 +1,564 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-"
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Process preprocessor output to move strings out of the image.
+
+This script is customized to be used in Chrome OS EC codebase, where functions
+cprintf(),cprints(), and cputs() are used to generate console output.
+
+Using this script requires a change in build process: instead of producing .o
+files from .c files, the process includes two extra steps: generate .E files
+instead of .o files, then process all .E files by this script, modifying some
+of them, producing a set of .Ep files, and then generate .o files from the .Ep
+files.
+
+A typical console output invocation in the EC codebase looks as follows:
+
+ cprintf(<chan>, <format string>, var args);
+
+This script is capable of processing invocations of up to 8 arguments (in fact
+in the Cr50 codebase at the time of introduction maximum number is 5). All
+generated .E files are given to this script as inputs. For each input file the
+script scans input lines and passes as is into the output lines which are not
+console output generating code.
+
+The format strings are extracted from console output generating lines and put
+into a dictionary, each line being the key, and the value an integer, the ever
+growing line number.
+
+After format strings are extracted and placed in the dictionary input lines
+are replaced with
+
+cmsg<X>(<chan>, <string index>, <param type map>, (uintptr_t)(param1), \
+      (uintptr_t)(param2)...)
+
+- X in cmsgX matches the number of parameters,
+- <chan> is passed as is,
+- <string index> is an integer to which the line number maps in the dictionary
+- <param type map> is a 32 bit integer, consisting of up to 8 fields, each
+           field describing the type of the parameter as it was specified in
+           the format string.
+
+Format specifications are interpreted as follows:
+
+%[Xcdux] - integer, 4 bytes
+%ll. - long, 8 bytes
+%s - string
+%pT - pointer, 4 bytes
+%pH - hex buf, 4 byte pointer to hex buf structure
+%pT - pointer to an 8 bytes value
+
+each of the above result in a different param type 4 bit nibble included in
+the <param type map>. A case when __func__ is used as a string parameter
+results in a unique parameter type, the function name is included in the
+dictionary, and just the index is passed as a cmsgX() parameter.
+
+When the original format string includes a 64 bit argument, the generated code
+creates a local 64 bit variable, assigns it the local variable to the argument
+and passes the address of the local value to the cmsgX invocation. This allows
+all cmsgX parameters to be of uintptr_t size.
+
+In cases when a string argument is __func__ this script saves the function
+name in the dictionary and coverts the parameter to the index, which is
+interpreted by the terminal to display the function name. A heuristic is used
+to determine the current function name: function starts with a line starting
+in the first column, containing an opening paren and not containing a
+semicolon. The word preceding the opening bracket is extracted and considered
+the current function name. Function scope ends with a } in the first column on
+its own line. This is not a very robust mechanism, but it seems to be adequate
+for this purpose.
+
+After all input files are processed, the format strings dictionary is
+converted into a list of strings such that each string is in the list at the
+location determined by the <string index>. This provides a means of mapping
+string indices into original format strings.
+
+The list is serialized, compressed and saved in a file as a flat blob. This
+file is used by the terminal program which interprets messages generated by
+the cmsgX() invocations and recreates the original string output on the
+terminal.
+
+There is a problem this scripts presents to a GNU make file: it requires
+multiple inputs to generate multiple outputs, but inputs are processed in a
+batch, only one script invocation is required.
+
+GNU Make does not seem to provide for such a concept, so this script is
+solving this problem as follows:
+
+When starting processing, obtain a file lock, and then verify that at least
+one of input files is newer that the previously generated strings blob, if it
+exists. If this is the case - process all input files, generate the blob and
+release the lock. If the blob exists and is newer - just release the lock and
+exit.
+
+As a result, when make is running this script is invoked as many times as
+there are .E files, but only one invocation results in processing, the rest
+just check that the newer blob is already there and exit.
+"""
+
+import argparse
+import fcntl
+import os
+import pickle
+import re
+import sys
+import zlib
+
+PRINTF_RX = r'[ \t]+cp(rint[fs]|uts)\('
+PTINTLINE = re.compile(PRINTF_RX)
+HASHLINE = re.compile(r'^# [0-9]+ ')
+TRAILING_SPACE = re.compile(r'\)[ \t]+;[ \t]*$')
+HEADER = re.compile(r'(%s[a-zA-Z_0-9]+),[ ]*' % PRINTF_RX)
+EOS = re.compile(r'[^\\\\]"[,\)]')
+FUNCTION = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_ \*]+\([^;]+$')
+INT_PARAM = re.compile(r'^[0-9.\-]*([l]{0,2}|z)[Xcdux]')
+STR_PARAM = re.compile(r'^[0-9.\-]*s')
+PTR_PARAM = re.compile(r'^p[hPT]')
+HEX_ESCAPE = re.compile(r'\\x[0-9a-fA-F]{2}')
+# Indexed by string, value - the string enum.
+FMT_DICT = {}
+
+# Parameter types
+PARAM_INT = 1
+PARAM_LONG = 2
+PARAM_STRING = 3
+PARAM_PTR = 4
+PARAM_HEX_BUF = 5
+PARAM_FUNC_NAME = 6
+PARAM_TIMESTAMP = 7
+
+def tokenize(params):
+    """Split C argument string into arguments (tokens).
+
+    Arguments within C string are comma separated, potentially include quoted
+    strings, which in turn could include escaped quotes and commas. There
+    could also be parens, possibly nested.
+
+    Only commas found outside quoted strings and parens should be considered
+    token separators.
+
+    Args:
+        params: A string, arguments given to say a C printf() invocation.
+
+    Returns:
+        A list of stings, parameters retrieved from 'params' with white space
+        stripped.
+    """
+
+    if ',' not in params:
+        return [params.strip(),]
+
+    tokens = []
+    parens_level = 0
+    in_q = False
+    in_double_q = False
+    token = ''
+    # need to skip all cases of commas in quotations or parens.
+    for i, char in enumerate(params):
+        char = params[i]
+        if char == ',' and parens_level == 0 and not in_q and not in_double_q:
+            tokens.append(token.strip())
+            token = ''
+            continue
+        token += char
+        if char in '{([':
+            parens_level += 1
+            continue
+        if char in '}])':
+            parens_level -= 1
+            continue
+        if char not in '\'"':
+            continue
+        if i > 0 and params[i - 1] == '\\':
+            continue
+        if char == '"':
+            in_double_q = not in_double_q
+            continue
+        in_q = not in_q
+    tokens.append(token.strip())
+    return tokens
+
+def drop_escapes(fmt):
+    'Return passed in string with escaped characters replaced by literals'
+    # First replace common single characters sequences.
+    unescapes = (("\\'", "'"),
+                 ('\\"', '"'),
+                 ('\\\\', '\\'),
+                 ('\\b', '\b'),
+                 ('\\a', '\a'),
+                 ('\\f', '\f'),
+                 ('\\n', '\n'),
+                 ('\\r', '\r'),
+                 ('\\t', '\t'),
+                 ('\\v', '\v'))
+    for (p, r) in unescapes:
+        fmt = fmt.replace(p, r)
+
+    # Then replace all hex values expressed as \xCC.
+    m = HEX_ESCAPE.search(fmt)
+    while m:
+        match = m.group(0)
+        c = chr(int('0x' + match[2:], 16))
+        fmt = HEX_ESCAPE.sub(c, fmt, count=1)
+        m = HEX_ESCAPE.search(fmt)
+    return fmt
+
+def generate_cmsg_line(fmt, params, fmt_blocks,
+                       channel, current_function):
+    """Given artifacts of a C line vararg line generate a cmsgX invocation line.
+
+    See details in the file docstring.
+
+    Args:
+        fmt: A string, the format string to be removed from the C source code.
+        params: A list of strings, the arguments of the fmt string from the
+               C source code.
+        fmt_blocks: A list of strings, the result of splitting the fmt
+              string at '%' characters. Could be obtained locally, but is
+              available from the caller.
+        channel: A string, name of the console channel the message is sent on.
+        current_function: A string, name of the C function this line is in.
+
+    Returns:
+        The cmsgX() string to replace the current source code string.
+        Also adds the format string into the dictionary, if it is not yet
+        there.
+    """
+    global FMT_DICT
+
+    # Since all parameters passed to cmsgX() are cast to (uintptr_t), uint64_t
+    # values could not be passed directly. If an int 64 parameter is
+    # encountered, a local variable is created to store the value, and the
+    # address of the variable is passed as the parameter to cmsgX().
+    #
+    # The doubles list below keeps track of these occurrences.
+    doubles = []
+    param_map = 0
+
+    fmt = drop_escapes(fmt)
+
+    # Either find the string among the previously encountered ones, or add it
+    # to the dictionary.
+    str_index = FMT_DICT.get(fmt, None)
+    if str_index is None:
+        str_index = len(FMT_DICT)
+        FMT_DICT[fmt] = str_index
+
+    # Now build a cmsgX invocation.
+    if params == ['']:
+        cmsg_index = 0
+    else:
+        cmsg_index = len(params)
+    generated_code = 'cmsg%d(%s, %d' % (cmsg_index, channel, str_index)
+    if cmsg_index == 0:
+        return generated_code + ');\n'
+    generated_code += ', PARAM^MAP' # Placeholder for parameters map.
+
+    # Preset the position of the first parameter descriptor in the parameter
+    # map, it will be incremented by 4 in the beginning of the loop.
+    position = -4
+    for block, param in zip(fmt_blocks, params):
+        position += 4
+        match = INT_PARAM.match(block)
+        if match:
+            double = match.group(1) == 'll'
+            if double:
+                param_map = param_map  + (PARAM_LONG << position)
+                var_name = 'll%d' % len(doubles)
+                doubles.append('%s = %s' % (var_name, param))
+                generated_code += ', (uintptr_t)&%s' % var_name
+            else:
+                param_map = param_map + (PARAM_INT << position)
+                generated_code += ', (uintptr_t)(%s)' % param
+            continue
+        if STR_PARAM.match(block):
+            if param == '__func__':
+                param_map = param_map + (PARAM_FUNC_NAME << position)
+                str_index = FMT_DICT.get(current_function, None)
+                if str_index is None:
+                    str_index = len(FMT_DICT)
+                    FMT_DICT[current_function] = str_index
+                generated_code += ', (uintptr_t)%d' % str_index
+                continue
+            param_map = param_map + (PARAM_STRING << position)
+            generated_code += ', (uintptr_t)(%s)' % param
+            continue
+        if PTR_PARAM.match(block):
+            types = {
+                'P': PARAM_PTR,
+                'T': PARAM_TIMESTAMP,
+                'h': PARAM_HEX_BUF
+            }
+            param_map = param_map + (types[block[1]] << position)
+            generated_code += ', (uintptr_t)(%s)' % param
+            continue
+
+    generated_code = generated_code.replace('PARAM^MAP', '%d' % param_map)
+    generated_code += ');'
+    if doubles:
+        extra = ['{', ]
+        for double in doubles:
+            extra.append('\tlong long %s;\n' % double)
+        generated_code = '\n'.join(extra) + generated_code + '\n}'
+
+    return generated_code + '\n'
+
+def process_ccprintf(line, current_function):
+    """Generate cmsgX line based on source cprint[fs] line.
+
+    Split the input line into elements, and invoke the function to convert the
+    elements into the matching cmstX() invocation.
+
+    Args:
+        line: A string, the source line to convert.
+        current_function: A string, the name of the C function 'l' comes from.
+
+    Returns:
+        The generated cmsgX() line.
+    """
+    # Let's split the string into components.
+    header = HEADER.search(line)
+    if not header:
+        return line # Must be not a valid ccprintf() invocation.
+
+    # If the line has some text before the function name, say 'return
+    # ccprintf...' save the text in preamble. If not - just set preamble to a
+    # single space.
+    start_ofs = header.span()[0]
+    if start_ofs != 0:
+        preamble = line[:start_ofs] + ' '
+    else:
+        preamble = ' '
+
+    # Not strictly necessary, but makes the output look neater, remove spaces
+    # after closing paren til newline.
+    line = TRAILING_SPACE.sub(');', line)
+
+    # Retrieve the channel name, guaranteed first argument.
+    channel = header.group(1).split('(')[1]
+
+    # Drop the 'cprintf(<channel>, ' header.
+    trailer = HEADER.sub('', line[start_ofs:])
+
+    # Find the end of the quoted format string.
+    quoted = EOS.search(trailer)
+    if not quoted:
+        if header.group(2) != 'uts':
+            sys.stderr.write('Unprocessed string: "%s"\n' % line)
+            return line
+        # This is a cputs() invocation with indirect string. Let's fix it by
+        # converting (param) into ("%s", param).
+        fmt = '%s'
+        params = [trailer.rstrip(';')[:-1],] # Strip ')[;]'.
+    else:
+        # Extract the fmt string, eliminate possible concatenations and drop
+        # starting and trailing double quotes.
+        fmt = trailer[:quoted.span()[0]] + quoted.group(0)
+        fmt = fmt.replace('" "', '')
+        fmt = fmt[1:-2]
+        if header.group(2) == 'rints':
+            fmt = '[^T' + fmt # terminal will add "]\n"
+
+        # Now get the list of parameters coming after the fmt string.
+        params = tokenize(trailer[quoted.span()[1]:].rstrip(';')[:-1])
+
+    # Elimitnate the %% cases and do not include the first element, it is
+    # irrelevant.
+    fmt_blocks = fmt.replace('%%', '').split('%')[1:]
+
+    if len(fmt_blocks) != len(params):
+        if fmt_blocks or params[0] != '':
+            sys.stderr.write('Unprocessed string: "%s"\n' % line)
+            print(trailer[quoted.span()[1]:].rstrip(';')[:-1])
+            return line
+
+    if len(params) > 8:
+        sys.stderr.write('Too many parameters: "%s"\n' % line)
+        return line
+
+    return preamble + generate_cmsg_line(fmt, params, fmt_blocks,
+                              channel, current_function)
+
+class LineProcessor(object):
+    """Process multiline source code strings.
+
+    The preprocessor output often generates C source code lines split in
+    multiple preprocessor output lines, in case there are macros in the
+    command line arguments, etc.
+
+    Before the line could be examined to be a printf() like invocation, and if
+    so converted into a cmsgX() line, the multiline preprocessor output needs
+    to be converted into a single code line.
+
+    This class allows to keep track of multiple preprocessor output lines.
+
+    Attributes:
+        partial_line: A string, concatenated preprocessor output lines
+           representing a single printf() like invocation statement.
+        current_function: A string, name of the function current lines belong
+           to.
+    """
+
+    def __init__(self):
+        self.partial_line = ''
+        self.current_function = None
+
+    def process_preprocessor_line(self, line):
+        """Process a preporcessor output line.
+
+        Examine the preprocessor output line to see if it falls into one of
+        three categories:
+
+        - A first function declaration line. Save the function name for future
+          reference.
+
+        - A closing '}' of a function - just drop the previously saved
+          function name.
+
+        - A line containing source code print statement
+          (cprintf/cprints/cputs). If the complete line is present - call the
+          parser to convert it into a cmsgX() invocation. If the line
+          preprocessor line is an incomplete C source line - keep processing
+          preprocessor lines until full source line is received and then call
+          the parser.
+
+        Args:
+            line: A string, a preprocessor output line.
+
+        Returns:
+            The input line if further processing is not required, or None, if
+            input line is not yet a full source code line, or the cmsgX()
+            invocation line, the result of converting a print statement line.
+        """
+        if FUNCTION.match(line):
+            # If this line looks like a first function definition line -
+            # retrieve the function name.
+            candidates = line.split()
+            for candidate in candidates:
+                if '(' in candidate:
+                    self.current_function = candidate.split('(')[0]
+        elif line.startswith('}'):
+            # If this is a '}' in the first column - we are not in a function
+            # scope any more.
+            self.current_function = None
+        if self.partial_line:
+            if HASHLINE.search(line):
+                # Drop preprocessor directives from within multiline C strings.
+                return None
+            self.partial_line += line.rstrip()
+            if ';' not in line:
+                # Still not a complete line, nothing to process yet.
+                return None
+            # Got a complete C source line, process it.
+            line = self.partial_line.rstrip()
+            self.partial_line = ''
+            return process_ccprintf(line, self.current_function)
+        if not PTINTLINE.search(line):
+            # If not a print statement - no need to worry, just pass it to the
+            # output as is.
+            return line
+        if line.startswith('int'):
+            # This is either the function prototype or the first line of the
+            # function definition, no need to convert either.
+            return line
+        if ');' not in line:
+            # This is a line with a print statement, but it is incomplete,
+            # start accumulating preprocessor lines.
+            self.partial_line = line.rstrip()
+            return None
+        # Process a full print statement line.
+        return process_ccprintf(line.rstrip(), self.current_function)
+
+def preobj_process(name, ext):
+    """Process a C preprocessor output file.
+
+    Given a preprocessor output file, generate a new file, with all print
+    statements replaced with cmsgX() invocations.
+
+    Args:
+        name: A string, name of the preprocessor output file to process.
+        ext: A string, the extension to use for the generated file.
+    """
+    line_processor = LineProcessor()
+    output = os.path.splitext(name)[0] + '.' + ext
+    with open(name, 'r') as in_file:
+        with open(output, 'w') as outf:
+            for lile in in_file.readlines():
+                processed = line_processor.process_preprocessor_line(lile)
+                if processed:
+                    outf.write(processed)
+
+def parse_args(argv):
+    """Prepare parser and parse command line arguments."""
+
+    prog = os.path.basename(argv[0])
+    parser = argparse.ArgumentParser(prog=prog)
+    parser.add_argument('-o', '--output')
+    parser.add_argument('-e', '--ext', default='Ep')
+    parser.add_argument('-l', '--lockfile', default='/tmp/%s.lockfile' % prog)
+    return parser.parse_known_args(argv)
+
+def generate_blob():
+    """Convert format strings dictionary into a blob.
+
+    First convert the format string dictionary into a list of strings, placed
+    in the list in locations matching their string index.
+
+    Then serialize and compress the list.
+
+    Returns:
+        A byte array, the compressed list of format lines.
+    """
+
+    strings = [''] * len(FMT_DICT)
+    for key, value in FMT_DICT.items():
+        strings[value] = key
+
+    dump = pickle.dumps('\0'.join(strings))
+    zipped = zlib.compress(dump, 9)
+    print('dump size %d, compressed size %d' % (len(dump), len(zipped)))
+    return zipped
+
+def main(argv):
+    """Main function.
+
+    To facilitate the use of this script in GNU Makefile, support multiple
+    concurrent invocations such that only one invocation does the processing.
+
+    First lock the lock file, then check if the blob to be generated is older
+    than any of the input files. If not - this means a different instance of
+    this script has already completed processing, just exit.
+    """
+    flags, files = parse_args(argv)
+
+    lfd = open(flags.lockfile, 'w')
+
+    fcntl.flock(lfd, fcntl.LOCK_EX)
+    try:
+        if os.path.exists(flags.output):
+            outp_stamp = os.stat(flags.output).st_mtime_ns
+
+            for e_file in files[1:]:
+                if os.stat(e_file).st_mtime_ns >= outp_stamp:
+                    break
+            else:
+                # Output file is newer than all inputs.
+                return
+
+        for e_file in files[1:]:
+            preobj_process(e_file, flags.ext)
+
+        zipped = generate_blob()
+        if flags.output:
+            outf = open(flags.output, 'wb')
+            outf.write(zipped)
+            outf.close()
+
+    finally:
+        fcntl.flock(lfd, fcntl.LOCK_UN)
+        lfd.close()
+
+if __name__ == '__main__':
+    main(sys.argv)