From b27d3f0d496b0358ab5c3bc2a8ecef5f863a635b Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 4 Jul 2014 14:18:08 +0000 Subject: Fix issue when overriding default chunk splitting rules When overriding a default chunk splitting rule, the use 'break' would mean that some of the other default chunk split rules might be ignored. This isn't ever what you want. I also clarified the current behaviour in a comment. I think in future we should add a mechanism to extend the default rules, as well as the current mechanism which allows only replacing them. But that is a separate issue. --- morphlib/artifactsplitrule.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index bc92e5fb..16b0e81b 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -230,10 +230,11 @@ def unify_chunk_matches(morphology): name = morphology['name'] for suffix, patterns in DEFAULT_CHUNK_RULES: ca_name = name + suffix - # Default rules are replaced by explicit ones - if ca_name in split_rules.artifacts: - break - split_rules.add(ca_name, FileMatch(patterns)) + # Explicit rules override the default rules. This is an all-or-nothing + # override: there is no way to extend the default split rules right now + # without duplicating them in the chunk morphology. + if ca_name not in split_rules.artifacts: + split_rules.add(ca_name, FileMatch(patterns)) return split_rules @@ -267,9 +268,11 @@ def unify_stratum_matches(morphology): for suffix, patterns in DEFAULT_STRATUM_RULES: sta_name = morphology['name'] + suffix - if sta_name in match_split_rules.artifacts: - break - match_split_rules.add(sta_name, ArtifactMatch(patterns)) + # Explicit rules override the default rules. This is an all-or-nothing + # override: there is no way to extend the default split rules right now + # without duplicating them in the chunk morphology. + if sta_name not in match_split_rules.artifacts: + match_split_rules.add(sta_name, ArtifactMatch(patterns)) # Construct a new SplitRules with the assignments before matches return SplitRules(itertools.chain(assignment_split_rules, -- cgit v1.2.1 From 9fa220986784bd9e4d63343ed0e22a2a8559ead7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 4 Jul 2014 15:37:45 +0100 Subject: Rewrite SplitRules.partition() docstring to be clearer to me --- morphlib/artifactsplitrule.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index 16b0e81b..125f5b93 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -150,9 +150,10 @@ class SplitRules(collections.Iterable): def partition(self, iterable): '''Match many files or artifacts. - It's the common case to take a bunch of filenames and determine - which artifact each should go to, so rather than implement this - logic in multiple places, it's here as a convenience method. + This function takes an iterable of file names, and groups them + using the rules that have been added to this object. + + This is a convenience function that uses the match() method internally. ''' -- cgit v1.2.1