Merging bug 21763 into default

3 files changed, 289 insertions, 7 deletions

diff --git a/README.extensions.md b/README.extensions.md
new file mode 100644
index 0000000..3a067ba
--- /dev/null
+++ b/README.extensions.md
@@ -0,0 +1,188 @@
+# Protocol extensions
+
+The `amqp_codegen.py` AMQP specification compiler has recently been
+enhanced to take more than a single specification file, which allows
+AMQP library authors to include extensions to the core protocol
+without needing to modify the core AMQP specification file as
+distributed.
+
+The compiler is invoked with the path to a single "main" specification
+document and zero or more paths to "extension" documents.
+
+The order of the extensions matters: any later class property
+definitions, for instance, are added to the list of definitions in
+order of appearance. In general, composition of extensions with a core
+specification document is therefore non-commutative.
+
+## The main document
+
+Written in the style of a
+[json-shapes](http://github.com/tonyg/json-shapes) schema:
+
+    DomainDefinition = _and(array_of(string()), array_length_equals(2));
+
+    ConstantDefinition = {
+        "name": string(),
+        "value": number(),
+        "class": optional(_or("soft-error", "hard-error"))
+    };
+
+    FieldDefinition = {
+        "name": string(),
+        "type": string(),
+        "default-value": optional(anything())
+    };
+
+    MethodDefinition = {
+        "name": string(),
+        "id": number(),
+        "arguments": array_of(FieldDefinition),
+        "synchronous": optional(boolean()),
+        "content": optional(boolean())
+    };
+
+    ClassDefinition = {
+        "name": string(),
+        "id": number(),
+        "methods": array_of(MethodDefinition),
+        "properties": optional(array_of(FieldDefinition))
+    };
+
+    MainDocument = {
+        "major-version": number(),
+        "minor-version": number(),
+        "revision": optional(number()),
+        "port": number(),
+        "domains": array_of(DomainDefinition),
+        "constants": array_of(ConstantDefinition),
+        "classes": array_of(ClassDefinition),
+    }
+
+Within a `FieldDefinition`, the keyword `domain` can be used instead
+of `type`, but `type` is preferred and `domain` is deprecated.
+
+Type names can either be a defined `domain` name or a built-in name
+from the following list:
+
+ - octet
+ - shortstr
+ - longstr
+ - short
+ - long
+ - longlong
+ - bit
+ - table
+ - timestamp
+
+Method and class IDs must be integers between 0 and 65535,
+inclusive. Note that there is no specific subset of the space reserved
+for experimental or site-local extensions, so be careful not to
+conflict with IDs used by the AMQP core specification.
+
+If the `synchronous` field of a `MethodDefinition` is missing, it is
+assumed to be `false`; the same applies to the `content` field.
+
+A `ConstantDefinition` with a `class` attribute is considered to be an
+error-code definition; otherwise, it is considered to be a
+straightforward numeric constant.
+
+## Extensions
+
+Written in the style of a
+[json-shapes](http://github.com/tonyg/json-shapes) schema, and
+referencing some of the type definitions given above:
+
+    ExtensionDocument = {
+        "extension": anything(),
+        "domains": array_of(DomainDefinition),
+        "constants": array_of(ConstantDefinition),
+        "classes": array_of(ClassDefinition)
+    };
+
+The `extension` keyword is used to describe the extension informally
+for human readers. Typically it will be a dictionary, with members
+such as:
+
+    {
+        "name": "The name of the extension",
+        "version": "1.0",
+        "copyright": "Copyright (C) 1234 Yoyodyne, Inc."
+    }
+
+## Merge behaviour
+
+In the case of conflicts between values specified in the main document
+and in any extension documents, type-specific merge operators are
+invoked.
+
+ - Any doubly-defined domain names are regarded as true
+   conflicts. Otherwise, all the domain definitions from all the main
+   and extension documents supplied to the compiler are merged into a
+   single dictionary.
+
+ - Constant definitions are treated as per domain names above,
+   *mutatis mutandis*.
+
+ - Classes and their methods are a little trickier: if an extension
+   defines a class with the same name as one previously defined, then
+   only the `methods` and `properties` fields of the extension's class
+   definition are attended to.
+
+    - Any doubly-defined method names or property names within a class
+      are treated as true conflicts.
+
+    - Properties defined in an extension are added to the end of the
+      extant property list for the class.
+
+   (Extensions are of course permitted to define brand new classes as
+   well as to extend existing ones.)
+
+ - Any other kind of conflict leads to a raised
+   `AmqpSpecFileMergeConflict` exception.
+
+## Invoking the spec compiler
+
+Your code generation code should invoke `amqp_codegen.do_main` with
+two functions as arguments: one for generating "header-file" text, and
+one for generating "implementation-file" text. The `do_main` function
+will parse the command-line arguments supplied when python was
+invoked.
+
+The command-line will be parsed as:
+
+    python your_codegen.py <headerorbody> <mainspec> [<extspec> ...] <outfile>
+
+where `<headerorbody>` is either the word `header` or the word `body`,
+to select which generation function is called by `do_main`. The
+`<mainspec>` and `<extspec>` arguments are file names of specification
+documents containing expressions in the syntax given above. The
+*final* argument on the command line, `<outfile>`, is the name of the
+source-code file to generate.
+
+Here's a tiny example of the layout of a code generation module that
+uses `amqp_codegen`:
+
+    import amqp_codegen
+
+    def generateHeader(specPath):
+        spec = amqp_codegen.AmqpSpec(specPath)
+        ...
+
+    def generateImpl(specPath):
+        spec = amqp_codegen.AmqpSpec(specPath)
+        ...
+
+    if __name__ == "__main__":
+        amqp_codegen.do_main(generateHeader, generateImpl)
+
+The reasons for this split, such as they are, are that
+
+ - many languages have separate "header"-type files (C and Erlang, to
+   name two)
+ - `Makefile`s often require separate rules for generating the two
+   kinds of file, but it's convenient to keep the generation code
+   together in a single python module
+
+The main reason things are laid out this way, however, is simply that
+it's an accident of the history of the code. We may change the API to
+`amqp_codegen` in future to clean things up a little.
diff --git a/amqp_codegen.py b/amqp_codegen.py
index 06c772a..2fcf9b6 100644
--- a/amqp_codegen.py
+++ b/amqp_codegen.py
@@ -58,10 +58,86 @@ def insert_base_types(d):
     for t in ['octet', 'shortstr', 'longstr', 'short', 'long',
               'longlong', 'bit', 'table', 'timestamp']:
         d[t] = t
+
+class AmqpSpecFileMergeConflict(Exception): pass
+
+def default_spec_value_merger(key, old, new):
+    if old is None or old == new:
+        return new
+    raise AmqpSpecFileMergeConflict(key, old, new)
+
+def extension_info_merger(key, old, new):
+    return old + [new]
+
+def domains_merger(key, old, new):
+    o = dict((k, v) for [k, v] in old)
+    for [k, v] in new:
+        if o.has_key(k):
+            raise AmqpSpecFileMergeConflict(key, old, new)
+        o[k] = v
+    return [[k, v] for (k, v) in o.iteritems()]
+
+def constants_merger(key, old, new):
+    o = dict((v["name"], v) for v in old)
+    for v in new:
+        if o.has_key(v["name"]):
+            raise AmqpSpecFileMergeConflict(key, old, new)
+        o[v["name"]] = v
+    return list(o.values())
+
+def methods_merger(classname, old, new):
+    o = dict((v["name"], v) for v in old)
+    for v in new:
+        if o.has_key(v["name"]):
+            raise AmqpSpecFileMergeConflict(("class-methods", classname), old, new)
+        o[v["name"]] = v
+    return list(o.values())
+
+def properties_merger(classname, old, new):
+    oldnames = set(v["name"] for v in old)
+    newnames = set(v["name"] for v in new)
+    clashes = oldnames.intersection(newnames)
+    if clashes:
+        raise AmqpSpecFileMergeConflict(("class-properties", classname), old, new)
+    return old + new
+
+def class_merger(old, new):
+    old["methods"] = methods_merger(old["name"], old["methods"], new["methods"])
+    old["properties"] = properties_merger(old["name"],
+                                          old.get("properties", []),
+                                          new.get("properties", []))
+    return old
+
+def classes_merger(key, old, new):
+    o = dict((v["name"], v) for v in old)
+    for v in new:
+        if o.has_key(v["name"]):
+            o[v["name"]] = class_merger(o[v["name"]], v)
+        else:
+            o[v["name"]] = v
+    return list(o.values())
+
+mergers = {
+    "extension": (extension_info_merger, []),
+    "domains": (domains_merger, []),
+    "constants": (constants_merger, []),
+    "classes": (classes_merger, []),
+}
+
+def merge_load_specs(filenames):
+    handles = [file(filename) for filename in filenames]
+    docs = [json.load(handle) for handle in handles]
+    spec = {}
+    for doc in docs:
+        for (key, value) in doc.iteritems():
+            (merger, default_value) = mergers.get(key, (default_spec_value_merger, None))
+            spec[key] = merger(key, spec.get(key, default_value), value)
+    for handle in handles: handle.close()
+    return spec
         
 class AmqpSpec:
-    def __init__(self, filename):
-        self.spec = json.load(file(filename))
+    def __init__(self, filenames):
+        self.spec = merge_load_specs(filenames)
 
         self.major = self.spec['major-version']
         self.minor = self.spec['minor-version']
@@ -175,13 +251,13 @@ def do_main(header_fn,body_fn):
         print >> sys.stderr , " %s header|body path_to_amqp_spec.json path_to_output_file" % (sys.argv[0])
         print >> sys.stderr , ""
         
-    def execute(fn, amqp_spec, out_file):
+    def execute(fn, amqp_specs, out_file):
         stdout = sys.stdout
         f = open(out_file, 'w')
         try:
             try:
                 sys.stdout = f
-                fn(amqp_spec)
+                fn(amqp_specs)
             except:
                 remove(out_file)
                 raise
@@ -189,14 +265,14 @@ def do_main(header_fn,body_fn):
             sys.stdout = stdout
             f.close()
 
-    if not len(sys.argv) == 4:
+    if len(sys.argv) < 4:
         usage()
         sys.exit(1)
     else:
         if sys.argv[1] == "header":
-            execute(header_fn, sys.argv[2], sys.argv[3])
+            execute(header_fn, sys.argv[2:-1], sys.argv[-1])
         elif sys.argv[1] == "body":
-            execute(body_fn, sys.argv[2], sys.argv[3])
+            execute(body_fn, sys.argv[2:-1], sys.argv[-1])
         else:
             usage()
             sys.exit(1)
diff --git a/demo_extension.json b/demo_extension.json
new file mode 100644
index 0000000..4a65dbe
--- /dev/null
+++ b/demo_extension.json
@@ -0,0 +1,18 @@
+{
+    "extension": {
+	"name": "demo",
+	"version": "1.0",
+	"copyright": "Copyright (C) 2009 LShift Ltd."
+    },
+    "domains": [
+	["foo-domain", "shortstr"]
+    ],
+    "constants": [
+	{"name": "FOO-CONSTANT", "value": 121212}
+    ],
+    "classes": [
+	{"name": "demo",
+	 "id": 555,
+	 "methods": [{"name": "one", "id": 1, "arguments": []}]}
+    ]
+}
\ No newline at end of file