From e423d17fccd8a8bd6f46aa69fe910e7cac77cd3b Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:49 +0900 Subject: tools: moveconfig: fix --dry-run option Since commit 96464badc794 ("moveconfig: Always run savedefconfig on the moved config"), --dry-run option is broken. The --dry-run option prevents the .config from being modified, but defconfig files might be updated by "make savedefconfig" regardless of the --dry-run option. Move the "if not self.options.dry_run" conditional to the correct place. Fixes 96464badc794 ("moveconfig: Always run savedefconfig on the moved config") Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 68631b7fae..fd98e41c5c 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -511,11 +511,10 @@ class KconfigParser: # Print log in one shot to not mix up logs from different threads. print log, - if not self.options.dry_run: - with open(dotconfig_path, 'a') as f: - for (action, value) in results: - if action == ACTION_MOVE: - f.write(value + '\n') + with open(dotconfig_path, 'a') as f: + for (action, value) in results: + if action == ACTION_MOVE: + f.write(value + '\n') os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf')) os.remove(autoconf_path) @@ -647,9 +646,9 @@ class Slot: return False if self.state == STATE_SAVEDEFCONFIG: - defconfig_path = os.path.join(self.build_dir, 'defconfig') - shutil.move(defconfig_path, - os.path.join('configs', self.defconfig)) + if not self.options.dry_run: + shutil.move(os.path.join(self.build_dir, 'defconfig'), + os.path.join('configs', self.defconfig)) self.state = STATE_IDLE return True -- cgit v1.2.1 From 6ff36d21746d61d7a2af08ea0bd973059d91d306 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:50 +0900 Subject: tools: moveconfig: rename update_defconfig() to update_dotconfig() Commit 96464badc794 ("moveconfig: Always run savedefconfig on the moved config") changed how defconfig files were updated. Since then, the function update_defconfig() does not modify defconfig files at all (instead, they are updated by "make savedefconfig"), so update_dotconfig() is a better fit for this function. Also, update the comment block to match the actual behavior. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index fd98e41c5c..9029287b10 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -459,13 +459,13 @@ class KconfigParser: return (action, value) - def update_defconfig(self, defconfig): - """Parse files for the config options and update the defconfig. + def update_dotconfig(self, defconfig): + """Parse files for the config options and update the .config. This function parses the given defconfig, the generated .config and include/autoconf.mk searching the target options. - Move the config option(s) to the defconfig or do nothing if unneeded. - Also, display the log to show what happened to this defconfig. + Move the config option(s) to the .config as needed. + Also, display the log to show what happened to the .config. Arguments: defconfig: defconfig name. @@ -632,7 +632,7 @@ class Slot: return True if self.state == STATE_AUTOCONF: - self.parser.update_defconfig(self.defconfig) + self.parser.update_dotconfig(self.defconfig) print ' %d defconfigs out of %d\r' % (self.num + 1, self.total), sys.stdout.flush() -- cgit v1.2.1 From ff8725bbe06b3b890beb2044c02ef332c8d028b4 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:51 +0900 Subject: tools: moveconfig: remove redundant else: after sys.exit() Nesting by "else:" is not generally useful after such statements as return, break, sys.exit(), etc. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 9029287b10..1332bd2830 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -623,13 +623,11 @@ class Slot: COLOR_LIGHT_CYAN, errout) if self.options.exit_on_error: sys.exit("Exit on error.") - else: - # If --exit-on-error flag is not set, - # skip this board and continue. - # Record the failed board. - self.failed_boards.append(self.defconfig) - self.state = STATE_IDLE - return True + # If --exit-on-error flag is not set, skip this board and continue. + # Record the failed board. + self.failed_boards.append(self.defconfig) + self.state = STATE_IDLE + return True if self.state == STATE_AUTOCONF: self.parser.update_dotconfig(self.defconfig) -- cgit v1.2.1 From 4b430c983a57146633f1b9a9ca5ed7289215763f Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:52 +0900 Subject: tools: moveconfig: check directory location before compilers We must ensure this tool is run from the top of source directory before calling update_cross_compile(). Otherwise, the following exception is thrown: Traceback (most recent call last): File "./moveconfig.py", line 918, in main() File "./moveconfig.py", line 908, in main update_cross_compile() File "./moveconfig.py", line 292, in update_cross_compile for arch in os.listdir('arch'): OSError: [Errno 2] No such file or directory: 'arch' Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 1332bd2830..ce8245a9f5 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -905,10 +905,10 @@ def main(): config_attrs = parse_recipe(args[0]) - update_cross_compile() - check_top_directory() + update_cross_compile() + if not options.cleanup_headers_only: move_config(config_attrs, options) -- cgit v1.2.1 From 90ed6cba51446f0e623ce076d6e565fad8909b46 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:53 +0900 Subject: tools: moveconfig: check compilers before starting defconfig walk Since commit 25400090b1e2 ("moveconfig: Print a message for missing compiler"), this tool parses an error message every time an error occurs during the process in order to detect missing compiler. Instead of that, we can look for compilers in the PATH environment only once before starting the defconfig walk. If a desired compiler is missing, "make include/config/auto.conf" will apparently fail for that architecture. So, the tool can just skip those board, showing "Compiler is missing. Do nothing.". Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 71 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index ce8245a9f5..7e916c2c9a 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -82,7 +82,12 @@ It looks like one of the followings: This config option was not found in the config header. Nothing to do. - - Failed to process. Skip. + - Compiler is missing. Do nothing. + The compiler specified for this architecture was not found + in your PATH environment. + (If -e option is passed, the tool exits immediately.) + + - Failed to process. An error occurred during processing this defconfig. Skipped. (If -e option is passed, the tool exits immediately on error.) @@ -272,7 +277,7 @@ def log_msg(color_enabled, color, defconfig, msg): return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \ color_text(color_enabled, color, msg) + '\n' -def update_cross_compile(): +def update_cross_compile(color_enabled): """Update per-arch CROSS_COMPILE via environment variables The default CROSS_COMPILE values are available @@ -286,6 +291,9 @@ def update_cross_compile(): export CROSS_COMPILE_ARM=... export CROSS_COMPILE_POWERPC=... + + Then, this function checks if specified compilers really exist in your + PATH environment. """ archs = [] @@ -299,8 +307,20 @@ def update_cross_compile(): for arch in archs: env = 'CROSS_COMPILE_' + arch.upper() cross_compile = os.environ.get(env) - if cross_compile: - CROSS_COMPILE[arch] = cross_compile + if not cross_compile: + cross_compile = CROSS_COMPILE.get(arch, '') + + for path in os.environ["PATH"].split(os.pathsep): + gcc_path = os.path.join(path, cross_compile + 'gcc') + if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): + break + else: + print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, + 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' + % (cross_compile, arch)) + cross_compile = None + + CROSS_COMPILE[arch] = cross_compile def cleanup_one_header(header_path, patterns, dry_run): """Clean regex-matched lines away from a file. @@ -387,6 +407,10 @@ class KconfigParser: Returns: A string storing the compiler prefix for the architecture. + Return a NULL string for architectures that do not require + compiler prefix (Sandbox and native build is the case). + Return None if the specified compiler is missing in your PATH. + Caller should distinguish '' and None. """ arch = '' cpu = '' @@ -400,13 +424,14 @@ class KconfigParser: if m: cpu = m.group(1) - assert arch, 'Error: arch is not defined in %s' % defconfig + if not arch: + return None # fix-up for aarch64 if arch == 'arm' and cpu == 'armv8': arch = 'aarch64' - return CROSS_COMPILE.get(arch, '') + return CROSS_COMPILE.get(arch, None) def parse_one_config(self, config_attr, defconfig_lines, autoconf_lines): """Parse .config, defconfig, include/autoconf.mk for one config. @@ -606,21 +631,12 @@ class Slot: return False if self.ps.poll() != 0: - errmsg = 'Failed to process.' - errout = self.ps.stderr.read() - if errout.find('gcc: command not found') != -1: - errmsg = 'Compiler not found (' - errmsg += color_text(self.options.color, COLOR_YELLOW, - self.cross_compile) - errmsg += color_text(self.options.color, COLOR_LIGHT_RED, - ')') - print >> sys.stderr, log_msg(self.options.color, - COLOR_LIGHT_RED, - self.defconfig, - errmsg), + print >> sys.stderr, log_msg(self.options.color, COLOR_LIGHT_RED, + self.defconfig, "Failed to process."), if self.options.verbose: print >> sys.stderr, color_text(self.options.color, - COLOR_LIGHT_CYAN, errout) + COLOR_LIGHT_CYAN, + self.ps.stderr.read()) if self.options.exit_on_error: sys.exit("Exit on error.") # If --exit-on-error flag is not set, skip this board and continue. @@ -651,15 +667,24 @@ class Slot: return True self.cross_compile = self.parser.get_cross_compile() + if self.cross_compile is None: + print >> sys.stderr, log_msg(self.options.color, COLOR_YELLOW, + self.defconfig, + "Compiler is missing. Do nothing."), + if self.options.exit_on_error: + sys.exit("Exit on error.") + # If --exit-on-error flag is not set, skip this board and continue. + # Record the failed board. + self.failed_boards.append(self.defconfig) + self.state = STATE_IDLE + return True + cmd = list(self.make_cmd) if self.cross_compile: cmd.append('CROSS_COMPILE=%s' % self.cross_compile) cmd.append('KCONFIG_IGNORE_DUPLICATES=1') cmd.append('include/config/auto.conf') - """This will be screen-scraped, so be sure the expected text will be - returned consistently on every machine by setting LANG=C""" self.ps = subprocess.Popen(cmd, stdout=self.devnull, - env=dict(os.environ, LANG='C'), stderr=subprocess.PIPE) self.state = STATE_AUTOCONF return False @@ -907,7 +932,7 @@ def main(): check_top_directory() - update_cross_compile() + update_cross_compile(options.color) if not options.cleanup_headers_only: move_config(config_attrs, options) -- cgit v1.2.1 From bd63e5baf9c1ec724374c6b6a5d88408479d34ca Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:54 +0900 Subject: tools: moveconfig: exit with error message for not clean directory When the source tree is not clean, this tool raises an exception with a message like follows: Traceback (most recent call last): File "tools/moveconfig.py", line 939, in main() File "tools/moveconfig.py", line 934, in main move_config(config_attrs, options) File "tools/moveconfig.py", line 808, in move_config while not slots.available(): File "tools/moveconfig.py", line 733, in available if slot.poll(): File "tools/moveconfig.py", line 645, in poll self.parser.update_dotconfig(self.defconfig) File "tools/moveconfig.py", line 503, in update_dotconfig with open(autoconf_path) as f: IOError: [Errno 2] No such file or directory: '/tmp/tmpDtzCgl/include/autoconf.mk' This does not explain what is wrong. Show an appropriate error message "source tree is not clean, please run 'make mrproper'" in such a situation. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 7e916c2c9a..0d64998e13 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -252,6 +252,12 @@ def check_top_directory(): if not os.path.exists(f): sys.exit('Please run at the top of source directory.') +def check_clean_directory(): + """Exit if the source tree is not clean.""" + for f in ('.config', 'include/config'): + if os.path.exists(f): + sys.exit("source tree is not clean, please run 'make mrproper'") + def get_make_cmd(): """Get the command name of GNU Make. @@ -932,6 +938,8 @@ def main(): check_top_directory() + check_clean_directory() + update_cross_compile(options.color) if not options.cleanup_headers_only: -- cgit v1.2.1 From c5e60fd4954ccbeee4d7cd783b338c4783604547 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:55 +0900 Subject: tools: moveconfig: increment number of processed files monotonically Currently, the progress " * defconfigs out of 1133" does not increase monotonically. Moreover, the number of processed defconfigs does not match the total number of defconfigs when this tool finishes, like: 1132 defconfigs out of 1133 Clean up headers? [y/n]: It looks like the task was not completed, and some users might feel upset about it. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 65 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 0d64998e13..caf2a82bf1 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -388,6 +388,29 @@ def cleanup_headers(config_attrs, dry_run): patterns, dry_run) ### classes ### +class Progress: + + """Progress Indicator""" + + def __init__(self, total): + """Create a new progress indicator. + + Arguments: + total: A number of defconfig files to process. + """ + self.current = 0 + self.total = total + + def inc(self): + """Increment the number of processed defconfig files.""" + + self.current += 1 + + def show(self): + """Display the progress.""" + print ' %d defconfigs out of %d\r' % (self.current, self.total), + sys.stdout.flush() + class KconfigParser: """A parser of .config and include/autoconf.mk.""" @@ -395,17 +418,19 @@ class KconfigParser: re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') - def __init__(self, config_attrs, options, build_dir): + def __init__(self, config_attrs, options, progress, build_dir): """Create a new parser. Arguments: config_attrs: A list of dictionaris, each of them includes the name, the type, and the default value of the target config. options: option flags. + progress: A progress indicator build_dir: Build directory. """ self.config_attrs = config_attrs self.options = options + self.progress = progress self.build_dir = build_dir def get_cross_compile(self): @@ -541,6 +566,7 @@ class KconfigParser: # Some threads are running in parallel. # Print log in one shot to not mix up logs from different threads. print log, + self.progress.show() with open(dotconfig_path, 'a') as f: for (action, value) in results: @@ -559,21 +585,24 @@ class Slot: for faster processing. """ - def __init__(self, config_attrs, options, devnull, make_cmd): + def __init__(self, config_attrs, options, progress, devnull, make_cmd): """Create a new process slot. Arguments: config_attrs: A list of dictionaris, each of them includes the name, the type, and the default value of the target config. options: option flags. + progress: A progress indicator. devnull: A file object of '/dev/null'. make_cmd: command name of GNU Make. """ self.options = options + self.progress = progress self.build_dir = tempfile.mkdtemp() self.devnull = devnull self.make_cmd = (make_cmd, 'O=' + self.build_dir) - self.parser = KconfigParser(config_attrs, options, self.build_dir) + self.parser = KconfigParser(config_attrs, options, progress, + self.build_dir) self.state = STATE_IDLE self.failed_boards = [] @@ -592,7 +621,7 @@ class Slot: pass shutil.rmtree(self.build_dir) - def add(self, defconfig, num, total): + def add(self, defconfig): """Assign a new subprocess for defconfig and add it to the slot. If the slot is vacant, create a new subprocess for processing the @@ -613,8 +642,6 @@ class Slot: stderr=subprocess.PIPE) self.defconfig = defconfig self.state = STATE_DEFCONFIG - self.num = num - self.total = total return True def poll(self): @@ -643,6 +670,8 @@ class Slot: print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_CYAN, self.ps.stderr.read()) + self.progress.inc() + self.progress.show() if self.options.exit_on_error: sys.exit("Exit on error.") # If --exit-on-error flag is not set, skip this board and continue. @@ -654,9 +683,6 @@ class Slot: if self.state == STATE_AUTOCONF: self.parser.update_dotconfig(self.defconfig) - print ' %d defconfigs out of %d\r' % (self.num + 1, self.total), - sys.stdout.flush() - """Save off the defconfig in a consistent way""" cmd = list(self.make_cmd) cmd.append('savedefconfig') @@ -669,6 +695,7 @@ class Slot: if not self.options.dry_run: shutil.move(os.path.join(self.build_dir, 'defconfig'), os.path.join('configs', self.defconfig)) + self.progress.inc() self.state = STATE_IDLE return True @@ -677,6 +704,8 @@ class Slot: print >> sys.stderr, log_msg(self.options.color, COLOR_YELLOW, self.defconfig, "Compiler is missing. Do nothing."), + self.progress.inc() + self.progress.show() if self.options.exit_on_error: sys.exit("Exit on error.") # If --exit-on-error flag is not set, skip this board and continue. @@ -704,22 +733,24 @@ class Slots: """Controller of the array of subprocess slots.""" - def __init__(self, config_attrs, options): + def __init__(self, config_attrs, options, progress): """Create a new slots controller. Arguments: config_attrs: A list of dictionaris containing the name, the type, and the default value of the target CONFIG. options: option flags. + progress: A progress indicator. """ self.options = options self.slots = [] devnull = get_devnull() make_cmd = get_make_cmd() for i in range(options.jobs): - self.slots.append(Slot(config_attrs, options, devnull, make_cmd)) + self.slots.append(Slot(config_attrs, options, progress, devnull, + make_cmd)) - def add(self, defconfig, num, total): + def add(self, defconfig): """Add a new subprocess if a vacant slot is found. Arguments: @@ -729,7 +760,7 @@ class Slots: Return True on success or False on failure """ for slot in self.slots: - if slot.add(defconfig, num, total): + if slot.add(defconfig): return True return False @@ -808,13 +839,14 @@ def move_config(config_attrs, options): for filename in fnmatch.filter(filenames, '*_defconfig'): defconfigs.append(os.path.join(dirpath, filename)) - slots = Slots(config_attrs, options) + progress = Progress(len(defconfigs)) + slots = Slots(config_attrs, options, progress) # Main loop to process defconfig files: # Add a new subprocess into a vacant slot. # Sleep if there is no available slot. - for i, defconfig in enumerate(defconfigs): - while not slots.add(defconfig, i, len(defconfigs)): + for defconfig in defconfigs: + while not slots.add(defconfig): while not slots.available(): # No available slot: sleep for a while time.sleep(SLEEP_TIME) @@ -823,6 +855,7 @@ def move_config(config_attrs, options): while not slots.empty(): time.sleep(SLEEP_TIME) + progress.show() print '' slots.show_failed_boards() -- cgit v1.2.1 From cc008299f8526dbbd2cb9485799218a67757c4e2 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:56 +0900 Subject: tools: moveconfig: do not rely on type and default value given by users Commit 96464badc794 ("moveconfig: Always run savedefconfig on the moved config") changed the work flow of this tool a lot from the original intention when this tool was designed first. Since then, before running this tool, users must edit the Kconfig to add the menu entries for the configs they are moving. It means users had already specified the type and the default value for each CONFIG via its Kconfig entry. Nevertheless, users are still required to dictate the same type and the default value in the input file. This is tedious to use. So, my idea here is to deprecate the latter. Before moving forward with it, there is one issue worth mentioning; since the savedefconfig re-sync was introduced, this tool has not been able to move bool options with "default y". Joe sent a patch to solve this problem about a year ago, but it was not applied for some reasons. Now, he came back with an updated patch, so this problem will be fixed soon. For other use cases, I see no reason to require redundant dictation in the input file. Instead, the tool can know the types and default values by parsing the .config file. This commit changes the tool to use the CONFIG names, but ignore the types and default values given by the input file. This commit also fixes one bug. Prior to this commit, it could not move an integer-typed CONFIG with value 1. For example, assume we are moving CONFIG_CONS_INDEX. Please note this is an integer type option. Many board headers define this CONFIG as 1. #define CONFIG_CONS_INDEX 1 It will be converted to CONFIG_CONS_INDEX=y and moved to include/autoconf.mk, by the tools/scripts/define2mk.sed. It will cause "make savedefconfig" to fail due to the type conflict. This commit takes care of it by detecting the type and converting the CONFIG value correctly. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 88 ++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index caf2a82bf1..96834c4f77 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -70,13 +70,17 @@ It looks like one of the followings: - Move 'CONFIG_... ' This config option was moved to the defconfig - - Default value 'CONFIG_...'. Do nothing. - The value of this option is the same as default. - We do not have to add it to the defconfig. - - - 'CONFIG_...' already exists in Kconfig. Do nothing. - This config option is already defined in Kconfig. - We do not need/want to touch it. + - CONFIG_... is not defined in Kconfig. Do nothing. + The entry for this CONFIG was not found in Kconfig. + There are two common cases: + - You forgot to create an entry for the CONFIG before running + this tool, or made a typo in a CONFIG passed to this tool. + - The entry was hidden due to unmet 'depends on'. + This is correct behavior. + + - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. + The define in the config header matched the one in Kconfig. + We do not need to touch it. - Undefined. Do nothing. This config option was not found in the config header. @@ -216,9 +220,8 @@ STATE_AUTOCONF = 2 STATE_SAVEDEFCONFIG = 3 ACTION_MOVE = 0 -ACTION_DEFAULT_VALUE = 1 -ACTION_ALREADY_EXIST = 2 -ACTION_UNDEFINED = 3 +ACTION_NO_ENTRY = 1 +ACTION_NO_CHANGE = 2 COLOR_BLACK = '0;30' COLOR_RED = '0;31' @@ -464,7 +467,7 @@ class KconfigParser: return CROSS_COMPILE.get(arch, None) - def parse_one_config(self, config_attr, defconfig_lines, autoconf_lines): + def parse_one_config(self, config_attr, dotconfig_lines, autoconf_lines): """Parse .config, defconfig, include/autoconf.mk for one config. This function looks for the config options in the lines from @@ -474,7 +477,7 @@ class KconfigParser: Arguments: config_attr: A dictionary including the name, the type, and the default value of the target config. - defconfig_lines: lines from the original defconfig file. + dotconfig_lines: lines from the .config file. autoconf_lines: lines from the include/autoconf.mk file. Returns: @@ -484,42 +487,40 @@ class KconfigParser: config = config_attr['config'] not_set = '# %s is not set' % config - if config_attr['type'] in ('bool', 'tristate') and \ - config_attr['default'] == 'n': - default = not_set - else: - default = config + '=' + config_attr['default'] - - for line in defconfig_lines: + for line in dotconfig_lines: line = line.rstrip() if line.startswith(config + '=') or line == not_set: - return (ACTION_ALREADY_EXIST, line) - - if config_attr['type'] in ('bool', 'tristate'): - value = not_set + old_val = line + break else: - value = '(undefined)' + return (ACTION_NO_ENTRY, config) for line in autoconf_lines: line = line.rstrip() if line.startswith(config + '='): - value = line + new_val = line break - - if value == default: - action = ACTION_DEFAULT_VALUE - elif value == '(undefined)': - action = ACTION_UNDEFINED else: - action = ACTION_MOVE + new_val = not_set + + if old_val == new_val: + return (ACTION_NO_CHANGE, new_val) + + # If this CONFIG is neither bool nor trisate + if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: + # tools/scripts/define2mk.sed changes '1' to 'y'. + # This is a problem if the CONFIG is int type. + # Check the type in Kconfig and handle it correctly. + if new_val[-2:] == '=y': + new_val = new_val[:-1] + '1' - return (action, value) + return (ACTION_MOVE, new_val) def update_dotconfig(self, defconfig): """Parse files for the config options and update the .config. - This function parses the given defconfig, the generated .config - and include/autoconf.mk searching the target options. + This function parses the generated .config and include/autoconf.mk + searching the target options. Move the config option(s) to the .config as needed. Also, display the log to show what happened to the .config. @@ -527,19 +528,18 @@ class KconfigParser: defconfig: defconfig name. """ - defconfig_path = os.path.join('configs', defconfig) dotconfig_path = os.path.join(self.build_dir, '.config') autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk') results = [] - with open(defconfig_path) as f: - defconfig_lines = f.readlines() + with open(dotconfig_path) as f: + dotconfig_lines = f.readlines() with open(autoconf_path) as f: autoconf_lines = f.readlines() for config_attr in self.config_attrs: - result = self.parse_one_config(config_attr, defconfig_lines, + result = self.parse_one_config(config_attr, dotconfig_lines, autoconf_lines) results.append(result) @@ -549,15 +549,13 @@ class KconfigParser: if action == ACTION_MOVE: actlog = "Move '%s'" % value log_color = COLOR_LIGHT_GREEN - elif action == ACTION_DEFAULT_VALUE: - actlog = "Default value '%s'. Do nothing." % value + elif action == ACTION_NO_ENTRY: + actlog = "%s is not defined in Kconfig. Do nothing." % value log_color = COLOR_LIGHT_BLUE - elif action == ACTION_ALREADY_EXIST: - actlog = "'%s' already defined in Kconfig. Do nothing." % value + elif action == ACTION_NO_CHANGE: + actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ + % value log_color = COLOR_LIGHT_PURPLE - elif action == ACTION_UNDEFINED: - actlog = "Undefined. Do nothing." - log_color = COLOR_DARK_GRAY else: sys.exit("Internal Error. This should not happen.") -- cgit v1.2.1 From b134bc13543f2627d3044310151459eeb904216c Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:57 +0900 Subject: tools: moveconfig: drop code for handling type and default value Now types and defalut values given by the input file are just ignored. Delete unnecessary code. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 105 +++++++++++++++------------------------------------- 1 file changed, 29 insertions(+), 76 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 96834c4f77..d54dd4e1c8 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -360,12 +360,11 @@ def cleanup_one_header(header_path, patterns, dry_run): if not i in matched: f.write(line) -def cleanup_headers(config_attrs, dry_run): +def cleanup_headers(configs, dry_run): """Delete config defines from board headers. Arguments: - config_attrs: A list of dictionaris, each of them includes the name, - the type, and the default value of the target config. + configs: A list of CONFIGs to remove. dry_run: make no changes, but still display log. """ while True: @@ -378,8 +377,7 @@ def cleanup_headers(config_attrs, dry_run): return patterns = [] - for config_attr in config_attrs: - config = config_attr['config'] + for config in configs: patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) @@ -421,17 +419,16 @@ class KconfigParser: re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') - def __init__(self, config_attrs, options, progress, build_dir): + def __init__(self, configs, options, progress, build_dir): """Create a new parser. Arguments: - config_attrs: A list of dictionaris, each of them includes the name, - the type, and the default value of the target config. + configs: A list of CONFIGs to move. options: option flags. progress: A progress indicator build_dir: Build directory. """ - self.config_attrs = config_attrs + self.configs = configs self.options = options self.progress = progress self.build_dir = build_dir @@ -467,7 +464,7 @@ class KconfigParser: return CROSS_COMPILE.get(arch, None) - def parse_one_config(self, config_attr, dotconfig_lines, autoconf_lines): + def parse_one_config(self, config, dotconfig_lines, autoconf_lines): """Parse .config, defconfig, include/autoconf.mk for one config. This function looks for the config options in the lines from @@ -475,8 +472,7 @@ class KconfigParser: which action should be taken for this defconfig. Arguments: - config_attr: A dictionary including the name, the type, - and the default value of the target config. + config: CONFIG name to parse. dotconfig_lines: lines from the .config file. autoconf_lines: lines from the include/autoconf.mk file. @@ -484,7 +480,6 @@ class KconfigParser: A tupple of the action for this defconfig and the line matched for the config. """ - config = config_attr['config'] not_set = '# %s is not set' % config for line in dotconfig_lines: @@ -538,8 +533,8 @@ class KconfigParser: with open(autoconf_path) as f: autoconf_lines = f.readlines() - for config_attr in self.config_attrs: - result = self.parse_one_config(config_attr, dotconfig_lines, + for config in self.configs: + result = self.parse_one_config(config, dotconfig_lines, autoconf_lines) results.append(result) @@ -583,12 +578,11 @@ class Slot: for faster processing. """ - def __init__(self, config_attrs, options, progress, devnull, make_cmd): + def __init__(self, configs, options, progress, devnull, make_cmd): """Create a new process slot. Arguments: - config_attrs: A list of dictionaris, each of them includes the name, - the type, and the default value of the target config. + configs: A list of CONFIGs to move. options: option flags. progress: A progress indicator. devnull: A file object of '/dev/null'. @@ -599,8 +593,7 @@ class Slot: self.build_dir = tempfile.mkdtemp() self.devnull = devnull self.make_cmd = (make_cmd, 'O=' + self.build_dir) - self.parser = KconfigParser(config_attrs, options, progress, - self.build_dir) + self.parser = KconfigParser(configs, options, progress, self.build_dir) self.state = STATE_IDLE self.failed_boards = [] @@ -731,12 +724,11 @@ class Slots: """Controller of the array of subprocess slots.""" - def __init__(self, config_attrs, options, progress): + def __init__(self, configs, options, progress): """Create a new slots controller. Arguments: - config_attrs: A list of dictionaris containing the name, the type, - and the default value of the target CONFIG. + configs: A list of CONFIGs to move. options: option flags. progress: A progress indicator. """ @@ -745,7 +737,7 @@ class Slots: devnull = get_devnull() make_cmd = get_make_cmd() for i in range(options.jobs): - self.slots.append(Slot(config_attrs, options, progress, devnull, + self.slots.append(Slot(configs, options, progress, devnull, make_cmd)) def add(self, defconfig): @@ -803,23 +795,18 @@ class Slots: for board in failed_boards: f.write(board + '\n') -def move_config(config_attrs, options): +def move_config(configs, options): """Move config options to defconfig files. Arguments: - config_attrs: A list of dictionaris, each of them includes the name, - the type, and the default value of the target config. + configs: A list of CONFIGs to move. options: option flags """ - if len(config_attrs) == 0: + if len(configs) == 0: print 'Nothing to do. exit.' sys.exit(0) - print 'Move the following CONFIG options (jobs: %d)' % options.jobs - for config_attr in config_attrs: - print ' %s (type: %s, default: %s)' % (config_attr['config'], - config_attr['type'], - config_attr['default']) + print 'Move %s (jobs: %d)' % (', '.join(configs), options.jobs) if options.defconfigs: defconfigs = [line.strip() for line in open(options.defconfigs)] @@ -838,7 +825,7 @@ def move_config(config_attrs, options): defconfigs.append(os.path.join(dirpath, filename)) progress = Progress(len(defconfigs)) - slots = Slots(config_attrs, options, progress) + slots = Slots(configs, options, progress) # Main loop to process defconfig files: # Add a new subprocess into a vacant slot. @@ -862,7 +849,7 @@ def bad_recipe(filename, linenum, msg): sys.exit("%s: line %d: error : " % (filename, linenum) + msg) def parse_recipe(filename): - """Parse the recipe file and retrieve the config attributes. + """Parse the recipe file and retrieve CONFIGs to move. This function parses the given recipe file and gets the name, the type, and the default value of the target config options. @@ -870,10 +857,9 @@ def parse_recipe(filename): Arguments: filename: path to file to be parsed. Returns: - A list of dictionaris, each of them includes the name, - the type, and the default value of the target config. + A list of CONFIGs to move. """ - config_attrs = [] + configs = [] linenum = 1 for line in open(filename): @@ -889,43 +875,10 @@ def parse_recipe(filename): if not config.startswith('CONFIG_'): config = 'CONFIG_' + config - # sanity check of default values - if type == 'bool': - if not default in ('y', 'n'): - bad_recipe(filename, linenum, - "default for bool type must be either y or n") - elif type == 'tristate': - if not default in ('y', 'm', 'n'): - bad_recipe(filename, linenum, - "default for tristate type must be y, m, or n") - elif type == 'string': - if default[0] != '"' or default[-1] != '"': - bad_recipe(filename, linenum, - "default for string type must be surrounded by double-quotations") - elif type == 'int': - try: - int(default) - except: - bad_recipe(filename, linenum, - "type is int, but default value is not decimal") - elif type == 'hex': - if len(default) < 2 or default[:2] != '0x': - bad_recipe(filename, linenum, - "default for hex type must be prefixed with 0x") - try: - int(default, 16) - except: - bad_recipe(filename, linenum, - "type is hex, but default value is not hexadecimal") - else: - bad_recipe(filename, linenum, - "unsupported type '%s'. type must be one of bool, tristate, string, int, hex" - % type) - - config_attrs.append({'config': config, 'type': type, 'default': default}) + configs.append(config) linenum += 1 - return config_attrs + return configs def main(): try: @@ -965,7 +918,7 @@ def main(): parser.print_usage() sys.exit(1) - config_attrs = parse_recipe(args[0]) + configs = parse_recipe(args[0]) check_top_directory() @@ -974,9 +927,9 @@ def main(): update_cross_compile(options.color) if not options.cleanup_headers_only: - move_config(config_attrs, options) + move_config(configs, options) - cleanup_headers(config_attrs, options.dry_run) + cleanup_headers(configs, options.dry_run) if __name__ == '__main__': main() -- cgit v1.2.1 From b6ef393ad78706398c0d7ec3358172af7fc8a66b Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:51:58 +0900 Subject: tools: moveconfig: allow to give CONFIG names as argument directly We still pass the input file with CONFIG name, type, default value in each line, but the last two fields are just ignored by the tool. So, let's deprecate the input file and allow users to give CONFIG names directly from the command line. The types and default values are automatically detected and handled nicely by the tool. Going forward, we can use this tool more easily like: tools/moveconfig.py CONFIG_FOO CONFIG_BAR Update the documentation and fix some typos I noticed while I was working on. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 108 ++++++++++------------------------------------------ 1 file changed, 20 insertions(+), 88 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index d54dd4e1c8..07136f56f5 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -17,44 +17,16 @@ This tool intends to help this tremendous work. Usage ----- -This tool takes one input file. (let's say 'recipe' file here.) -The recipe describes the list of config options you want to move. -Each line takes the form: - -(the fields must be separated with whitespaces.) - - is the name of config option. - - is the type of the option. It must be one of bool, tristate, -string, int, and hex. - - is the default value of the option. It must be appropriate -value corresponding to the option type. It must be either y or n for -the bool type. Tristate options can also take m (although U-Boot has -not supported the module feature). - -You can add two or more lines in the recipe file, so you can move -multiple options at once. - -Let's say, for example, you want to move CONFIG_CMD_USB and -CONFIG_SYS_TEXT_BASE. - -The type should be bool, hex, respectively. So, the recipe file -should look like this: - - $ cat recipe - CONFIG_CMD_USB bool n - CONFIG_SYS_TEXT_BASE hex 0x00000000 - -Next you must edit the Kconfig to add the menu entries for the configs +First, you must edit the Kconfig to add the menu entries for the configs you are moving. -And then run this tool giving the file name of the recipe +And then run this tool giving CONFIG names you want to move. +For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, +simply type as follows: - $ tools/moveconfig.py recipe + $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE -The tool walks through all the defconfig files to move the config -options specified by the recipe file. +The tool walks through all the defconfig files and move the given CONFIGs. The log is also displayed on the terminal. @@ -103,19 +75,19 @@ It just uses the regex method, so you should not rely on it. Just in case, please do 'git diff' to see what happened. -How does it works? ------------------- +How does it work? +----------------- This tool runs configuration and builds include/autoconf.mk for every defconfig. The config options defined in Kconfig appear in the .config file (unless they are hidden because of unmet dependency.) On the other hand, the config options defined by board headers are seen in include/autoconf.mk. The tool looks for the specified options in both -of them to decide the appropriate action for the options. If the option -is found in the .config or the value is the same as the specified default, -the option does not need to be touched. If the option is found in -include/autoconf.mk, but not in the .config, and the value is different -from the default, the tools adds the option to the defconfig. +of them to decide the appropriate action for the options. If the given +config option is found in the .config, but its value does not match the +one from the board header, the config option in the .config is replaced +with the define in the board header. Then, the .config is synced by +"make savedefconfig" and the defconfig is updated with it. For faster processing, this tool handles multi-threading. It creates separate build directories where the out-of-tree build is run. The @@ -148,7 +120,7 @@ Available options Specify a file containing a list of defconfigs to move -n, --dry-run - Peform a trial run that does not make any changes. It is useful to + Perform a trial run that does not make any changes. It is useful to see what is going to happen before one actually runs it. -e, --exit-on-error @@ -844,42 +816,6 @@ def move_config(configs, options): print '' slots.show_failed_boards() -def bad_recipe(filename, linenum, msg): - """Print error message with the file name and the line number and exit.""" - sys.exit("%s: line %d: error : " % (filename, linenum) + msg) - -def parse_recipe(filename): - """Parse the recipe file and retrieve CONFIGs to move. - - This function parses the given recipe file and gets the name, - the type, and the default value of the target config options. - - Arguments: - filename: path to file to be parsed. - Returns: - A list of CONFIGs to move. - """ - configs = [] - linenum = 1 - - for line in open(filename): - tokens = line.split() - if len(tokens) != 3: - bad_recipe(filename, linenum, - "%d fields in this line. Each line must contain 3 fields" - % len(tokens)) - - (config, type, default) = tokens - - # prefix the option name with CONFIG_ if missing - if not config.startswith('CONFIG_'): - config = 'CONFIG_' + config - - configs.append(config) - linenum += 1 - - return configs - def main(): try: cpu_count = multiprocessing.cpu_count() @@ -904,21 +840,17 @@ def main(): help='the number of jobs to run simultaneously') parser.add_option('-v', '--verbose', action='store_true', default=False, help='show any build errors as boards are built') - parser.usage += ' recipe_file\n\n' + \ - 'The recipe_file should describe config options you want to move.\n' + \ - 'Each line should contain config_name, type, default_value\n\n' + \ - 'Example:\n' + \ - 'CONFIG_FOO bool n\n' + \ - 'CONFIG_BAR int 100\n' + \ - 'CONFIG_BAZ string "hello"\n' + parser.usage += ' CONFIG ...' - (options, args) = parser.parse_args() + (options, configs) = parser.parse_args() - if len(args) != 1: + if len(configs) == 0: parser.print_usage() sys.exit(1) - configs = parse_recipe(args[0]) + # prefix the option name with CONFIG_ if missing + configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config + for config in configs ] check_top_directory() -- cgit v1.2.1 From 1f16992ee93595fa840aff55bdb722185cc31ca5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:00 +0900 Subject: tools: moveconfig: compute file paths just once The paths to .config, include/autoconf.mk, include/config/auto.conf are not changed during the defconfig walk. Compute them only once when a new class instance is created. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 07136f56f5..7dd9d8c582 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -403,7 +403,10 @@ class KconfigParser: self.configs = configs self.options = options self.progress = progress - self.build_dir = build_dir + self.dotconfig = os.path.join(build_dir, '.config') + self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') + self.config_autoconf = os.path.join(build_dir, 'include', 'config', + 'auto.conf') def get_cross_compile(self): """Parse .config file and return CROSS_COMPILE. @@ -417,8 +420,7 @@ class KconfigParser: """ arch = '' cpu = '' - dotconfig = os.path.join(self.build_dir, '.config') - for line in open(dotconfig): + for line in open(self.dotconfig): m = self.re_arch.match(line) if m: arch = m.group(1) @@ -495,14 +497,12 @@ class KconfigParser: defconfig: defconfig name. """ - dotconfig_path = os.path.join(self.build_dir, '.config') - autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk') results = [] - with open(dotconfig_path) as f: + with open(self.dotconfig) as f: dotconfig_lines = f.readlines() - with open(autoconf_path) as f: + with open(self.autoconf) as f: autoconf_lines = f.readlines() for config in self.configs: @@ -533,13 +533,13 @@ class KconfigParser: print log, self.progress.show() - with open(dotconfig_path, 'a') as f: + with open(self.dotconfig, 'a') as f: for (action, value) in results: if action == ACTION_MOVE: f.write(value + '\n') - os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf')) - os.remove(autoconf_path) + os.remove(self.config_autoconf) + os.remove(self.autoconf) class Slot: -- cgit v1.2.1 From 522e8dcb4c1d7a64fc19d1545d71b5ee356640bf Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:01 +0900 Subject: tools: moveconfig: move log output code out of Kconfig Parser class This will help further improvement/clean-up. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 7dd9d8c582..97d2fafadc 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -391,18 +391,16 @@ class KconfigParser: re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') - def __init__(self, configs, options, progress, build_dir): + def __init__(self, configs, options, build_dir): """Create a new parser. Arguments: configs: A list of CONFIGs to move. options: option flags. - progress: A progress indicator build_dir: Build directory. """ self.configs = configs self.options = options - self.progress = progress self.dotconfig = os.path.join(build_dir, '.config') self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') self.config_autoconf = os.path.join(build_dir, 'include', 'config', @@ -491,10 +489,12 @@ class KconfigParser: This function parses the generated .config and include/autoconf.mk searching the target options. Move the config option(s) to the .config as needed. - Also, display the log to show what happened to the .config. Arguments: defconfig: defconfig name. + + Returns: + Return log string """ results = [] @@ -528,11 +528,6 @@ class KconfigParser: log += log_msg(self.options.color, log_color, defconfig, actlog) - # Some threads are running in parallel. - # Print log in one shot to not mix up logs from different threads. - print log, - self.progress.show() - with open(self.dotconfig, 'a') as f: for (action, value) in results: if action == ACTION_MOVE: @@ -541,6 +536,8 @@ class KconfigParser: os.remove(self.config_autoconf) os.remove(self.autoconf) + return log + class Slot: """A slot to store a subprocess. @@ -565,7 +562,7 @@ class Slot: self.build_dir = tempfile.mkdtemp() self.devnull = devnull self.make_cmd = (make_cmd, 'O=' + self.build_dir) - self.parser = KconfigParser(configs, options, progress, self.build_dir) + self.parser = KconfigParser(configs, options, self.build_dir) self.state = STATE_IDLE self.failed_boards = [] @@ -644,7 +641,7 @@ class Slot: return True if self.state == STATE_AUTOCONF: - self.parser.update_dotconfig(self.defconfig) + self.log = self.parser.update_dotconfig(self.defconfig) """Save off the defconfig in a consistent way""" cmd = list(self.make_cmd) @@ -658,7 +655,11 @@ class Slot: if not self.options.dry_run: shutil.move(os.path.join(self.build_dir, 'defconfig'), os.path.join('configs', self.defconfig)) + # Some threads are running in parallel. + # Print log in one shot to not mix up logs from different threads. + print self.log, self.progress.inc() + self.progress.show() self.state = STATE_IDLE return True @@ -812,7 +813,6 @@ def move_config(configs, options): while not slots.empty(): time.sleep(SLEEP_TIME) - progress.show() print '' slots.show_failed_boards() -- cgit v1.2.1 From 1d085568b3def83b541ab0d7ee550c52f638eab5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:02 +0900 Subject: tools: moveconfig: display log atomically in more readable format Before this commit, the log was displayed in the format: : : : When we move multiple CONFIGs at the same time, we see as many strings as actions for every defconfig, which is redundant information. Moreover, since normal log and error log are displayed separately, Messages from different threads could be mixed, like this: : : : : : This commit makes sure to call "print" once a defconfig, which enables atomic logging for each defconfig. It also makes it possible to refactor the log format as follows: Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 73 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 97d2fafadc..5906bd03f6 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -30,13 +30,17 @@ The tool walks through all the defconfig files and move the given CONFIGs. The log is also displayed on the terminal. -Each line is printed in the format - : +The log is printed for each defconfig as follows: - is the name of the defconfig -(without the suffix _defconfig). + + + + + ... - shows what the tool did for that defconfig. + is the name of the defconfig. + + shows what the tool did for that defconfig. It looks like one of the followings: - Move 'CONFIG_... ' @@ -249,15 +253,13 @@ def get_make_cmd(): def color_text(color_enabled, color, string): """Return colored string.""" if color_enabled: - return '\033[' + color + 'm' + string + '\033[0m' + # LF should not be surrounded by the escape sequence. + # Otherwise, additional whitespace or line-feed might be printed. + return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' + for s in string.split('\n') ]) else: return string -def log_msg(color_enabled, color, defconfig, msg): - """Return the formated line for the log.""" - return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \ - color_text(color_enabled, color, msg) + '\n' - def update_cross_compile(color_enabled): """Update per-arch CROSS_COMPILE via environment variables @@ -483,7 +485,7 @@ class KconfigParser: return (ACTION_MOVE, new_val) - def update_dotconfig(self, defconfig): + def update_dotconfig(self): """Parse files for the config options and update the .config. This function parses the generated .config and include/autoconf.mk @@ -526,7 +528,7 @@ class KconfigParser: else: sys.exit("Internal Error. This should not happen.") - log += log_msg(self.options.color, log_color, defconfig, actlog) + log += color_text(self.options.color, log_color, actlog) + '\n' with open(self.dotconfig, 'a') as f: for (action, value) in results: @@ -602,6 +604,7 @@ class Slot: stderr=subprocess.PIPE) self.defconfig = defconfig self.state = STATE_DEFCONFIG + self.log = '' return True def poll(self): @@ -624,14 +627,12 @@ class Slot: return False if self.ps.poll() != 0: - print >> sys.stderr, log_msg(self.options.color, COLOR_LIGHT_RED, - self.defconfig, "Failed to process."), + self.log += color_text(self.options.color, COLOR_LIGHT_RED, + "Failed to process.\n") if self.options.verbose: - print >> sys.stderr, color_text(self.options.color, - COLOR_LIGHT_CYAN, - self.ps.stderr.read()) - self.progress.inc() - self.progress.show() + self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, + self.ps.stderr.read()) + self.show_log(sys.stderr) if self.options.exit_on_error: sys.exit("Exit on error.") # If --exit-on-error flag is not set, skip this board and continue. @@ -641,7 +642,7 @@ class Slot: return True if self.state == STATE_AUTOCONF: - self.log = self.parser.update_dotconfig(self.defconfig) + self.log += self.parser.update_dotconfig() """Save off the defconfig in a consistent way""" cmd = list(self.make_cmd) @@ -655,21 +656,15 @@ class Slot: if not self.options.dry_run: shutil.move(os.path.join(self.build_dir, 'defconfig'), os.path.join('configs', self.defconfig)) - # Some threads are running in parallel. - # Print log in one shot to not mix up logs from different threads. - print self.log, - self.progress.inc() - self.progress.show() + self.show_log() self.state = STATE_IDLE return True self.cross_compile = self.parser.get_cross_compile() if self.cross_compile is None: - print >> sys.stderr, log_msg(self.options.color, COLOR_YELLOW, - self.defconfig, - "Compiler is missing. Do nothing."), - self.progress.inc() - self.progress.show() + self.log += color_text(self.options.color, COLOR_YELLOW, + "Compiler is missing. Do nothing.\n") + self.show_log(sys.stderr) if self.options.exit_on_error: sys.exit("Exit on error.") # If --exit-on-error flag is not set, skip this board and continue. @@ -688,6 +683,22 @@ class Slot: self.state = STATE_AUTOCONF return False + def show_log(self, file=sys.stdout): + """Display log along with progress. + + Arguments: + file: A file object to which the log string is sent. + """ + # output at least 30 characters to hide the "* defconfigs out of *". + log = self.defconfig.ljust(30) + '\n' + + log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) + # Some threads are running in parallel. + # Print log atomically to not mix up logs from different threads. + print >> file, log + self.progress.inc() + self.progress.show() + def get_failed_boards(self): """Returns a list of failed boards (defconfigs) in this slot. """ -- cgit v1.2.1 From 4efef998cda69edb162ea61e4a14f564133457d1 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:03 +0900 Subject: tools: moveconfig: refactor code to go back to idle state Move similar code to finish() function. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 5906bd03f6..84ad16d876 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -632,13 +632,7 @@ class Slot: if self.options.verbose: self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, self.ps.stderr.read()) - self.show_log(sys.stderr) - if self.options.exit_on_error: - sys.exit("Exit on error.") - # If --exit-on-error flag is not set, skip this board and continue. - # Record the failed board. - self.failed_boards.append(self.defconfig) - self.state = STATE_IDLE + self.finish(False) return True if self.state == STATE_AUTOCONF: @@ -656,21 +650,14 @@ class Slot: if not self.options.dry_run: shutil.move(os.path.join(self.build_dir, 'defconfig'), os.path.join('configs', self.defconfig)) - self.show_log() - self.state = STATE_IDLE + self.finish(True) return True self.cross_compile = self.parser.get_cross_compile() if self.cross_compile is None: self.log += color_text(self.options.color, COLOR_YELLOW, "Compiler is missing. Do nothing.\n") - self.show_log(sys.stderr) - if self.options.exit_on_error: - sys.exit("Exit on error.") - # If --exit-on-error flag is not set, skip this board and continue. - # Record the failed board. - self.failed_boards.append(self.defconfig) - self.state = STATE_IDLE + self.finish(False) return True cmd = list(self.make_cmd) @@ -683,11 +670,12 @@ class Slot: self.state = STATE_AUTOCONF return False - def show_log(self, file=sys.stdout): - """Display log along with progress. + def finish(self, success): + """Display log along with progress and go to the idle state. Arguments: - file: A file object to which the log string is sent. + success: Should be True when the defconfig was processed + successfully, or False when it fails. """ # output at least 30 characters to hide the "* defconfigs out of *". log = self.defconfig.ljust(30) + '\n' @@ -695,9 +683,18 @@ class Slot: log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) # Some threads are running in parallel. # Print log atomically to not mix up logs from different threads. - print >> file, log + print >> (sys.stdout if success else sys.stderr), log + + if not success: + if self.options.exit_on_error: + sys.exit("Exit on error.") + # If --exit-on-error flag is not set, skip this board and continue. + # Record the failed board. + self.failed_boards.append(self.defconfig) + self.progress.inc() self.progress.show() + self.state = STATE_IDLE def get_failed_boards(self): """Returns a list of failed boards (defconfigs) in this slot. -- cgit v1.2.1 From 7fb0bacd389b9b041164965d31a8c50e8616f7a0 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:04 +0900 Subject: tools: moveconfig: skip savedefconfig if .config was not updated If no CONFIG option is moved to the .config, no need to sync the defconfig file. This accelerates the processing by skipping needless "make savedefconfig". Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 84ad16d876..e32104412e 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -496,10 +496,13 @@ class KconfigParser: defconfig: defconfig name. Returns: - Return log string + Return a tuple of (updated flag, log string). + The "updated flag" is True if the .config was updated, False + otherwise. The "log string" shows what happend to the .config. """ results = [] + updated = False with open(self.dotconfig) as f: dotconfig_lines = f.readlines() @@ -534,11 +537,12 @@ class KconfigParser: for (action, value) in results: if action == ACTION_MOVE: f.write(value + '\n') + updated = True os.remove(self.config_autoconf) os.remove(self.autoconf) - return log + return (updated, log) class Slot: @@ -614,8 +618,11 @@ class Slot: If the configuration is successfully finished, assign a new subprocess to build include/autoconf.mk. If include/autoconf.mk is generated, invoke the parser to - parse the .config and the include/autoconf.mk, and then set the - slot back to the idle state. + parse the .config and the include/autoconf.mk, moving + config options to the .config as needed. + If the .config was updated, run "make savedefconfig" to sync + it, update the original defconfig, and then set the slot back + to the idle state. Returns: Return True if the subprocess is terminated, False otherwise @@ -636,8 +643,12 @@ class Slot: return True if self.state == STATE_AUTOCONF: - self.log += self.parser.update_dotconfig() + (updated, log) = self.parser.update_dotconfig() + self.log += log + if not updated: + self.finish(True) + return True """Save off the defconfig in a consistent way""" cmd = list(self.make_cmd) cmd.append('savedefconfig') -- cgit v1.2.1 From c1c4d0f056f0c0fc8ad6397ad3ad870948d8951e Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:05 +0900 Subject: tools: moveconfig: display log when savedefconfig occurs Now, "make savedefconfig" does not always happen. Display the log when it happens. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index e32104412e..9c8ab19f0a 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -649,7 +649,8 @@ class Slot: if not updated: self.finish(True) return True - """Save off the defconfig in a consistent way""" + self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, + "Syncing by savedefconfig...\n") cmd = list(self.make_cmd) cmd.append('savedefconfig') self.ps = subprocess.Popen(cmd, stdout=self.devnull, -- cgit v1.2.1 From 5da4f857beac1cf859aefac3cf2ae980115ec1ae Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:06 +0900 Subject: tools: moveconfig: report when CONFIGs are removed by savedefconfig This is a rare case, but there is still possibility that some CONFIG is moved to the .config, but it is removed by "make savedefconfig". (For example, it happens when the specified CONFIG has no prompt in the Kconfig entry, i.e. it is not user-configurable.) It might be an unexpected case. So, display the log in this case (in yellow color to gain user's attention if --color option is given). Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 9c8ab19f0a..4881ec539b 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -407,6 +407,7 @@ class KconfigParser: self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') self.config_autoconf = os.path.join(build_dir, 'include', 'config', 'auto.conf') + self.defconfig = os.path.join(build_dir, 'defconfig') def get_cross_compile(self): """Parse .config file and return CROSS_COMPILE. @@ -539,11 +540,35 @@ class KconfigParser: f.write(value + '\n') updated = True + self.results = results os.remove(self.config_autoconf) os.remove(self.autoconf) return (updated, log) + def check_defconfig(self): + """Check the defconfig after savedefconfig + + Returns: + Return additional log if moved CONFIGs were removed again by + 'make savedefconfig'. + """ + + log = '' + + with open(self.defconfig) as f: + defconfig_lines = f.readlines() + + for (action, value) in self.results: + if action != ACTION_MOVE: + continue + if not value + '\n' in defconfig_lines: + log += color_text(self.options.color, COLOR_YELLOW, + "'%s' was removed by savedefconfig.\n" % + value) + + return log + class Slot: """A slot to store a subprocess. @@ -659,6 +684,7 @@ class Slot: return False if self.state == STATE_SAVEDEFCONFIG: + self.log += self.parser.check_defconfig() if not self.options.dry_run: shutil.move(os.path.join(self.build_dir, 'defconfig'), os.path.join('configs', self.defconfig)) -- cgit v1.2.1 From c8e1b10d07b1dbbf3cbf882ac270476b964d05ff Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:07 +0900 Subject: tools: moveconfig: report when defconfig is updated There are various factors that determine if the given defconfig is updated, and it is probably what users are more interested in. Show the log when the defconfig is updated. Also, copy the file only when the file content was really updated to avoid changing the time stamp needlessly. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 4881ec539b..a8531f1e51 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -147,6 +147,7 @@ To see the complete list of supported options, run """ +import filecmp import fnmatch import multiprocessing import optparse @@ -685,9 +686,16 @@ class Slot: if self.state == STATE_SAVEDEFCONFIG: self.log += self.parser.check_defconfig() - if not self.options.dry_run: - shutil.move(os.path.join(self.build_dir, 'defconfig'), - os.path.join('configs', self.defconfig)) + orig_defconfig = os.path.join('configs', self.defconfig) + new_defconfig = os.path.join(self.build_dir, 'defconfig') + updated = not filecmp.cmp(orig_defconfig, new_defconfig) + + if updated: + self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, + "defconfig was updated.\n") + + if not self.options.dry_run and updated: + shutil.move(new_defconfig, orig_defconfig) self.finish(True) return True -- cgit v1.2.1 From 8513dc048598bba15f75ab3fbe257159cb27007a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:08 +0900 Subject: tools: moveconfig: add --force-sync option Now, this tools invokes "make savedefconfig" only when it needs to do so, but there might be cases where a user wants the tool to do savedefconfig forcibly, for example, some defconfigs were already out of sync and the user wants to fix it as well. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index a8531f1e51..95a73569a1 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -131,6 +131,11 @@ Available options Exit immediately if Make exits with a non-zero status while processing a defconfig file. + -s, --force-sync + Do "make savedefconfig" forcibly for all the defconfig files. + If not specified, "make savedefconfig" only occurs for cases + where at least one CONFIG was moved. + -H, --headers-only Only cleanup the headers; skip the defconfig processing @@ -672,11 +677,15 @@ class Slot: (updated, log) = self.parser.update_dotconfig() self.log += log - if not updated: + if not self.options.force_sync and not updated: self.finish(True) return True - self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, - "Syncing by savedefconfig...\n") + if updated: + self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, + "Syncing by savedefconfig...\n") + else: + self.log += "Syncing by savedefconfig (forced by option)...\n" + cmd = list(self.make_cmd) cmd.append('savedefconfig') self.ps = subprocess.Popen(cmd, stdout=self.devnull, @@ -887,6 +896,8 @@ def main(): parser.add_option('-e', '--exit-on-error', action='store_true', default=False, help='exit immediately on any error') + parser.add_option('-s', '--force-sync', action='store_true', default=False, + help='force sync by savedefconfig') parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', action='store_true', default=False, help='only cleanup the headers') -- cgit v1.2.1 From 6a9f79f712dfce7c451c26120ffc112d3e1a88e7 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 May 2016 15:52:09 +0900 Subject: tools: moveconfig: allow to run without any CONFIG specified I found "tools/moveconfig -s" might be useful for defconfig re-sync. I could optimize it for re-sync if I wanted, but I do not want to make the code complex for this feature. Signed-off-by: Masahiro Yamada Reviewed-by: Joe Hershberger --- tools/moveconfig.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 95a73569a1..eee28b37a6 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -839,10 +839,13 @@ def move_config(configs, options): options: option flags """ if len(configs) == 0: - print 'Nothing to do. exit.' - sys.exit(0) - - print 'Move %s (jobs: %d)' % (', '.join(configs), options.jobs) + if options.force_sync: + print 'No CONFIG is specified. You are probably syncing defconfigs.', + else: + print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', + else: + print 'Move ' + ', '.join(configs), + print '(jobs: %d)\n' % options.jobs if options.defconfigs: defconfigs = [line.strip() for line in open(options.defconfigs)] @@ -909,7 +912,7 @@ def main(): (options, configs) = parser.parse_args() - if len(configs) == 0: + if len(configs) == 0 and not options.force_sync: parser.print_usage() sys.exit(1) @@ -926,7 +929,8 @@ def main(): if not options.cleanup_headers_only: move_config(configs, options) - cleanup_headers(configs, options.dry_run) + if configs: + cleanup_headers(configs, options.dry_run) if __name__ == '__main__': main() -- cgit v1.2.1 From e307fa9d89e0f12e900c0fe80c76453fc1c7c2b6 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 8 Jun 2016 11:47:37 +0900 Subject: tools: moveconfig: make Slot.poll() more readable with helper methods The Slot.poll() method is already complicated and a new feature we are going to add will make it more difficult to understand the execution flow. Refactor it with helper methods, .handle_error(), .do_defconfig(), .do_autoconf(), .do_savedefconfig, and .update_defconfig(). Signed-off-by: Masahiro Yamada --- tools/moveconfig.py | 110 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 43 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index eee28b37a6..9bbcead2eb 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -633,13 +633,10 @@ class Slot: """ if self.state != STATE_IDLE: return False - cmd = list(self.make_cmd) - cmd.append(defconfig) - self.ps = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE) + self.defconfig = defconfig - self.state = STATE_DEFCONFIG self.log = '' + self.do_defconfig() return True def poll(self): @@ -665,55 +662,46 @@ class Slot: return False if self.ps.poll() != 0: - self.log += color_text(self.options.color, COLOR_LIGHT_RED, - "Failed to process.\n") - if self.options.verbose: - self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, - self.ps.stderr.read()) - self.finish(False) - return True + self.handle_error() + elif self.state == STATE_DEFCONFIG: + self.do_autoconf() + elif self.state == STATE_AUTOCONF: + self.do_savedefconfig() + elif self.state == STATE_SAVEDEFCONFIG: + self.update_defconfig() + else: + sys.exit("Internal Error. This should not happen.") - if self.state == STATE_AUTOCONF: - (updated, log) = self.parser.update_dotconfig() - self.log += log + return True if self.state == STATE_IDLE else False - if not self.options.force_sync and not updated: - self.finish(True) - return True - if updated: - self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, - "Syncing by savedefconfig...\n") - else: - self.log += "Syncing by savedefconfig (forced by option)...\n" + def handle_error(self): + """Handle error cases.""" - cmd = list(self.make_cmd) - cmd.append('savedefconfig') - self.ps = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE) - self.state = STATE_SAVEDEFCONFIG - return False + self.log += color_text(self.options.color, COLOR_LIGHT_RED, + "Failed to process.\n") + if self.options.verbose: + self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, + self.ps.stderr.read()) + self.finish(False) - if self.state == STATE_SAVEDEFCONFIG: - self.log += self.parser.check_defconfig() - orig_defconfig = os.path.join('configs', self.defconfig) - new_defconfig = os.path.join(self.build_dir, 'defconfig') - updated = not filecmp.cmp(orig_defconfig, new_defconfig) + def do_defconfig(self): + """Run 'make _defconfig' to create the .config file.""" - if updated: - self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, - "defconfig was updated.\n") + cmd = list(self.make_cmd) + cmd.append(self.defconfig) + self.ps = subprocess.Popen(cmd, stdout=self.devnull, + stderr=subprocess.PIPE) + self.state = STATE_DEFCONFIG - if not self.options.dry_run and updated: - shutil.move(new_defconfig, orig_defconfig) - self.finish(True) - return True + def do_autoconf(self): + """Run 'make include/config/auto.conf'.""" self.cross_compile = self.parser.get_cross_compile() if self.cross_compile is None: self.log += color_text(self.options.color, COLOR_YELLOW, "Compiler is missing. Do nothing.\n") self.finish(False) - return True + return cmd = list(self.make_cmd) if self.cross_compile: @@ -723,7 +711,43 @@ class Slot: self.ps = subprocess.Popen(cmd, stdout=self.devnull, stderr=subprocess.PIPE) self.state = STATE_AUTOCONF - return False + + def do_savedefconfig(self): + """Update the .config and run 'make savedefconfig'.""" + + (updated, log) = self.parser.update_dotconfig() + self.log += log + + if not self.options.force_sync and not updated: + self.finish(True) + return + if updated: + self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, + "Syncing by savedefconfig...\n") + else: + self.log += "Syncing by savedefconfig (forced by option)...\n" + + cmd = list(self.make_cmd) + cmd.append('savedefconfig') + self.ps = subprocess.Popen(cmd, stdout=self.devnull, + stderr=subprocess.PIPE) + self.state = STATE_SAVEDEFCONFIG + + def update_defconfig(self): + """Update the input defconfig and go back to the idle state.""" + + self.log += self.parser.check_defconfig() + orig_defconfig = os.path.join('configs', self.defconfig) + new_defconfig = os.path.join(self.build_dir, 'defconfig') + updated = not filecmp.cmp(orig_defconfig, new_defconfig) + + if updated: + self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, + "defconfig was updated.\n") + + if not self.options.dry_run and updated: + shutil.move(new_defconfig, orig_defconfig) + self.finish(True) def finish(self, success): """Display log along with progress and go to the idle state. -- cgit v1.2.1 From f2dae751fbaa029cd3b1fe13cb4710f5c52cbc8e Mon Sep 17 00:00:00 2001 From: Joe Hershberger Date: Fri, 10 Jun 2016 14:53:29 -0500 Subject: tools: moveconfig: Fix another typo Signed-off-by: Joe Hershberger Reviewed-by: Masahiro Yamada --- tools/moveconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 9bbcead2eb..d2744c16eb 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -608,7 +608,7 @@ class Slot: This function makes sure the temporary directory is cleaned away even if Python suddenly dies due to error. It should be done in here - because it is guranteed the destructor is always invoked when the + because it is guaranteed the destructor is always invoked when the instance of the class gets unreferenced. If the subprocess is still running, wait until it finishes. -- cgit v1.2.1 From 06cc1d36d0e5ae4954f60bce78f6ac2117c3b357 Mon Sep 17 00:00:00 2001 From: Joe Hershberger Date: Fri, 10 Jun 2016 14:53:30 -0500 Subject: tools: moveconfig: New color used for changed defconfig The old color blends in with similar messages and makes them not stand out. Signed-off-by: Joe Hershberger Reviewed-by: Masahiro Yamada --- tools/moveconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index d2744c16eb..12a145c0a6 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -742,7 +742,7 @@ class Slot: updated = not filecmp.cmp(orig_defconfig, new_defconfig) if updated: - self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, + self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, "defconfig was updated.\n") if not self.options.dry_run and updated: -- cgit v1.2.1 From 6b96c1a1ca9e86e0f377f18dc470d054c4509c8a Mon Sep 17 00:00:00 2001 From: Joe Hershberger Date: Fri, 10 Jun 2016 14:53:32 -0500 Subject: tools: moveconfig: Add a new --git-ref option This option allows the 'make autoconf.mk' step to run against a former repo state, while the savedefconfig step runs against the current repo state. This is convenient for the case where something in the Kconfig has changed such that the defconfig is no longer complete with the new Kconfigs. This feature allows the .config to be built assuming those old Kconfigs, but then savedefconfig based on the new state of the Kconfigs. If in doubt, always specify this switch. It will always do the right thing even if not required, but if it was required and you don't use it, the moved configs will be incorrect. When not using this switch, you must very carefully evaluate that all moved configs are correct. Signed-off-by: Joe Hershberger Reviewed-by: Masahiro Yamada --- tools/moveconfig.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index 12a145c0a6..5e5ca06d8f 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -143,6 +143,14 @@ Available options Specify the number of threads to run simultaneously. If not specified, the number of threads is the same as the number of CPU cores. + -r, --git-ref + Specify the git ref to clone for building the autoconf.mk. If unspecified + use the CWD. This is useful for when changes to the Kconfig affect the + default values and you want to capture the state of the defconfig from + before that change was in effect. If in doubt, specify a ref pre-Kconfig + changes (use HEAD if Kconfig changes are not committed). Worst case it will + take a bit longer to run, but will always do the right thing. + -v, --verbose Show any build errors as boards are built @@ -584,7 +592,7 @@ class Slot: for faster processing. """ - def __init__(self, configs, options, progress, devnull, make_cmd): + def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): """Create a new process slot. Arguments: @@ -593,12 +601,15 @@ class Slot: progress: A progress indicator. devnull: A file object of '/dev/null'. make_cmd: command name of GNU Make. + reference_src_dir: Determine the true starting config state from this + source tree. """ self.options = options self.progress = progress self.build_dir = tempfile.mkdtemp() self.devnull = devnull self.make_cmd = (make_cmd, 'O=' + self.build_dir) + self.reference_src_dir = reference_src_dir self.parser = KconfigParser(configs, options, self.build_dir) self.state = STATE_IDLE self.failed_boards = [] @@ -636,6 +647,7 @@ class Slot: self.defconfig = defconfig self.log = '' + self.use_git_ref = True if self.options.git_ref else False self.do_defconfig() return True @@ -664,9 +676,16 @@ class Slot: if self.ps.poll() != 0: self.handle_error() elif self.state == STATE_DEFCONFIG: - self.do_autoconf() + if self.options.git_ref and not self.use_git_ref: + self.do_savedefconfig() + else: + self.do_autoconf() elif self.state == STATE_AUTOCONF: - self.do_savedefconfig() + if self.use_git_ref: + self.use_git_ref = False + self.do_defconfig() + else: + self.do_savedefconfig() elif self.state == STATE_SAVEDEFCONFIG: self.update_defconfig() else: @@ -689,6 +708,9 @@ class Slot: cmd = list(self.make_cmd) cmd.append(self.defconfig) + if self.use_git_ref: + cmd.append('-C') + cmd.append(self.reference_src_dir) self.ps = subprocess.Popen(cmd, stdout=self.devnull, stderr=subprocess.PIPE) self.state = STATE_DEFCONFIG @@ -708,6 +730,9 @@ class Slot: cmd.append('CROSS_COMPILE=%s' % self.cross_compile) cmd.append('KCONFIG_IGNORE_DUPLICATES=1') cmd.append('include/config/auto.conf') + if self.use_git_ref: + cmd.append('-C') + cmd.append(self.reference_src_dir) self.ps = subprocess.Popen(cmd, stdout=self.devnull, stderr=subprocess.PIPE) self.state = STATE_AUTOCONF @@ -784,13 +809,15 @@ class Slots: """Controller of the array of subprocess slots.""" - def __init__(self, configs, options, progress): + def __init__(self, configs, options, progress, reference_src_dir): """Create a new slots controller. Arguments: configs: A list of CONFIGs to move. options: option flags. progress: A progress indicator. + reference_src_dir: Determine the true starting config state from this + source tree. """ self.options = options self.slots = [] @@ -798,7 +825,7 @@ class Slots: make_cmd = get_make_cmd() for i in range(options.jobs): self.slots.append(Slot(configs, options, progress, devnull, - make_cmd)) + make_cmd, reference_src_dir)) def add(self, defconfig): """Add a new subprocess if a vacant slot is found. @@ -855,6 +882,24 @@ class Slots: for board in failed_boards: f.write(board + '\n') +class WorkDir: + def __init__(self): + """Create a new working directory.""" + self.work_dir = tempfile.mkdtemp() + + def __del__(self): + """Delete the working directory + + This function makes sure the temporary directory is cleaned away + even if Python suddenly dies due to error. It should be done in here + because it is guaranteed the destructor is always invoked when the + instance of the class gets unreferenced. + """ + shutil.rmtree(self.work_dir) + + def get(self): + return self.work_dir + def move_config(configs, options): """Move config options to defconfig files. @@ -871,6 +916,21 @@ def move_config(configs, options): print 'Move ' + ', '.join(configs), print '(jobs: %d)\n' % options.jobs + reference_src_dir = '' + + if options.git_ref: + work_dir = WorkDir() + reference_src_dir = work_dir.get() + print "Cloning git repo to a separate work directory..." + subprocess.check_output(['git', 'clone', os.getcwd(), '.'], + cwd=reference_src_dir) + print "Checkout '%s' to build the original autoconf.mk." % \ + subprocess.check_output(['git', 'rev-parse', '--short', + options.git_ref]).strip() + subprocess.check_output(['git', 'checkout', options.git_ref], + stderr=subprocess.STDOUT, + cwd=reference_src_dir) + if options.defconfigs: defconfigs = [line.strip() for line in open(options.defconfigs)] for i, defconfig in enumerate(defconfigs): @@ -888,7 +948,7 @@ def move_config(configs, options): defconfigs.append(os.path.join(dirpath, filename)) progress = Progress(len(defconfigs)) - slots = Slots(configs, options, progress) + slots = Slots(configs, options, progress, reference_src_dir) # Main loop to process defconfig files: # Add a new subprocess into a vacant slot. @@ -930,6 +990,8 @@ def main(): help='only cleanup the headers') parser.add_option('-j', '--jobs', type='int', default=cpu_count, help='the number of jobs to run simultaneously') + parser.add_option('-r', '--git-ref', type='string', + help='the git ref to clone for building the autoconf.mk') parser.add_option('-v', '--verbose', action='store_true', default=False, help='show any build errors as boards are built') parser.usage += ' CONFIG ...' -- cgit v1.2.1 From cd51878e34f6beef8ec7b66886e5a23a64b88653 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 11 Jun 2016 18:44:10 +0900 Subject: tools: fix define2mk.sed to not add quotes around negative integers The sed script, tools/scripts/define2mk.sed, converts config defines from C headers into include/autoconf.mk for the use in Makefiles. I found the tool adds quotes around negative integer values. For example, at the point of the v2016.07-rc1 tag, include/configs/microblaze-generic.h defines #define CONFIG_BOOTDELAY -1 /* -1 disables auto-boot */ Because it is an integer option, it should be converted to: CONFIG_BOOTDELAY=-1 But, the script actually converts it to: CONFIG_BOOTDELAY="-1" This is a fatal problem for the tools/moveconfig.py because it parses include/autoconf.mk for the config defines from the board headers. CONFIG_BOOTDELAY="-1" is considered as a string type option and it is dropped due to the type mismatch from the entry in Kconfig. This commit fixes the script so that the tools/moveconfig.py can correctly convert integer options with a negative value. Signed-off-by: Masahiro Yamada --- tools/scripts/define2mk.sed | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/scripts/define2mk.sed b/tools/scripts/define2mk.sed index c641edfb01..0f00285f36 100644 --- a/tools/scripts/define2mk.sed +++ b/tools/scripts/define2mk.sed @@ -22,6 +22,8 @@ s/=\(..*\)/="\1"/; # but remove again from decimal numbers s/="\([0-9][0-9]*\)"/=\1/; + # ... and from negative decimal numbers + s/="\(-[1-9][0-9]*\)"/=\1/; # ... and from hex numbers s/="\(0[Xx][0-9a-fA-F][0-9a-fA-F]*\)"/=\1/; # ... and from configs defined from other configs -- cgit v1.2.1 From 2c15534f6b5b2157b7c93b6d7c59e26989afb069 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 11 Jun 2016 18:44:08 +0900 Subject: ARM: stm32: remove unused CONFIG_AUTOBOOT At this point, this is not referenced from anywhere, so remove it (but it will be re-added later for a different meaning). Signed-off-by: Masahiro Yamada --- include/configs/stm32f429-discovery.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/configs/stm32f429-discovery.h b/include/configs/stm32f429-discovery.h index 8bbe580ebf..f05c1aacaf 100644 --- a/include/configs/stm32f429-discovery.h +++ b/include/configs/stm32f429-discovery.h @@ -82,8 +82,6 @@ "bootcmd_romfs=setenv bootargs ${bootargs} ${bootargs_romfs};" \ "bootm 0x08044000 - 0x08042000\0" -#define CONFIG_AUTOBOOT - /* * Command line configuration. */ -- cgit v1.2.1 From 3865b9ebe73eb6246ff2ab6ae7957f507eb539f7 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 11 Jun 2016 18:44:09 +0900 Subject: autoboot: follow-up cleanup after CONFIG_BOOTDELAY moves Tidy up garbage left by commit bb597c0eeb7e ("common: bootdelay: move CONFIG_BOOTDELAY into a Kconfig option"). Signed-off-by: Masahiro Yamada --- include/configs/bfin_adi_common.h | 5 ----- include/configs/mx6_common.h | 3 --- include/configs/mx7_common.h | 3 --- include/configs/omap3_cairo.h | 2 -- 4 files changed, 13 deletions(-) diff --git a/include/configs/bfin_adi_common.h b/include/configs/bfin_adi_common.h index 13076072e4..502ddad96b 100644 --- a/include/configs/bfin_adi_common.h +++ b/include/configs/bfin_adi_common.h @@ -97,11 +97,6 @@ /* * Env Settings */ -#ifndef CONFIG_BOOTDELAY -# if (CONFIG_BFIN_BOOT_MODE == BFIN_BOOT_UART) -# else -# endif -#endif #ifndef CONFIG_BOOTCOMMAND # define CONFIG_BOOTCOMMAND "run ramboot" #endif diff --git a/include/configs/mx6_common.h b/include/configs/mx6_common.h index 27f38f497e..fb49322f3b 100644 --- a/include/configs/mx6_common.h +++ b/include/configs/mx6_common.h @@ -59,9 +59,6 @@ #endif #define CONFIG_SYS_LOAD_ADDR CONFIG_LOADADDR -#ifndef CONFIG_BOOTDELAY -#endif - /* allow to overwrite serial and ethaddr */ #define CONFIG_ENV_OVERWRITE #define CONFIG_CONS_INDEX 1 diff --git a/include/configs/mx7_common.h b/include/configs/mx7_common.h index 5e5656d480..fbc6de6283 100644 --- a/include/configs/mx7_common.h +++ b/include/configs/mx7_common.h @@ -34,9 +34,6 @@ #define CONFIG_LOADADDR 0x80800000 #define CONFIG_SYS_TEXT_BASE 0x87800000 -#ifndef CONFIG_BOOTDELAY -#endif - /* allow to overwrite serial and ethaddr */ #define CONFIG_ENV_OVERWRITE #define CONFIG_CONS_INDEX 1 diff --git a/include/configs/omap3_cairo.h b/include/configs/omap3_cairo.h index 96c3c4b958..82e0d5000c 100644 --- a/include/configs/omap3_cairo.h +++ b/include/configs/omap3_cairo.h @@ -73,8 +73,6 @@ #define CONFIG_NAND_OMAP_GPMC #define CONFIG_SYS_MAX_NAND_DEVICE 1 /* Max number of NAND */ /* devices */ -/* override default CONFIG_BOOTDELAY */ - #define CONFIG_EXTRA_ENV_SETTINGS \ "machid=ffffffff\0" \ "fdt_high=0x87000000\0" \ -- cgit v1.2.1