From dc89be70621c1f5ea94c91cb94d4a2cb0e32c626 Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Wed, 7 Oct 2009 14:55:02 +0100 Subject: Infrastructure for supporting piecewise spec extension. --- amqp_codegen.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/amqp_codegen.py b/amqp_codegen.py index 06c772a..bd42950 100644 --- a/amqp_codegen.py +++ b/amqp_codegen.py @@ -58,10 +58,31 @@ def insert_base_types(d): for t in ['octet', 'shortstr', 'longstr', 'short', 'long', 'longlong', 'bit', 'table', 'timestamp']: d[t] = t + +class AmqpSpecFileMergeConflict(Exception): pass + +def default_spec_value_merger(key, old, new): + if old is None or old == new: + return new + raise AmqpSpecFileMergeConflict((key, old, new)) + +mergers = { +} + +def merge_load_specs(filenames): + handles = [file(filename) for filename in filenames] + docs = [json.load(handle) for handle in handles] + spec = {} + for doc in docs: + for (key, value) in doc.iteritems(): + (merger, default_value) = mergers.get(key, (default_spec_value_merger, None)) + spec[key] = merger(key, spec.get(key, default_value), value) + for handle in handles: handle.close() + return spec class AmqpSpec: - def __init__(self, filename): - self.spec = json.load(file(filename)) + def __init__(self, filenames): + self.spec = merge_load_specs(filenames) self.major = self.spec['major-version'] self.minor = self.spec['minor-version'] @@ -175,13 +196,13 @@ def do_main(header_fn,body_fn): print >> sys.stderr , " %s header|body path_to_amqp_spec.json path_to_output_file" % (sys.argv[0]) print >> sys.stderr , "" - def execute(fn, amqp_spec, out_file): + def execute(fn, amqp_specs, out_file): stdout = sys.stdout f = open(out_file, 'w') try: try: sys.stdout = f - fn(amqp_spec) + fn(amqp_specs) except: remove(out_file) raise @@ -189,14 +210,14 @@ def do_main(header_fn,body_fn): sys.stdout = stdout f.close() - if not len(sys.argv) == 4: + if len(sys.argv) < 4: usage() sys.exit(1) else: if sys.argv[1] == "header": - execute(header_fn, sys.argv[2], sys.argv[3]) + execute(header_fn, sys.argv[2:-1], sys.argv[-1]) elif sys.argv[1] == "body": - execute(body_fn, sys.argv[2], sys.argv[3]) + execute(body_fn, sys.argv[2:-1], sys.argv[-1]) else: usage() sys.exit(1) -- cgit v1.2.1 From 2827b11a8020d2d59f813528d476d85969eb574c Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Wed, 7 Oct 2009 16:11:54 +0100 Subject: Extensions mergers. --- amqp_codegen.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/amqp_codegen.py b/amqp_codegen.py index bd42950..1ade01f 100644 --- a/amqp_codegen.py +++ b/amqp_codegen.py @@ -66,7 +66,52 @@ def default_spec_value_merger(key, old, new): return new raise AmqpSpecFileMergeConflict((key, old, new)) +def extension_info_merger(key, old, new): + return old + [new] + +def domains_merger(key, old, new): + o = dict((k, v) for [k, v] in old) + for [k, v] in new: + if o.has_key(k): + raise AmqpSpecFileMergeConflict(key, old, new) + o[k] = v + return [[k, v] for (k, v) in o.iteritems()] + +def constants_merger(key, old, new): + o = dict((v["name"], v) for v in old) + for v in new: + if o.has_key(v["name"]): + raise AmqpSpecFileMergeConflict(key, old, new) + o[v["name"]] = v + return list(o.values()) + +def methods_merger(classname, old, new): + o = dict((v["name"], v) for v in old) + for v in new: + if o.has_key(v["name"]): + raise AmqpSpecFileMergeConflict(("class", classname), old, new) + o[v["name"]] = v + return list(o.values()) + +def class_merger(old, new): + old["methods"] = methods_merger(old["name"], old["methods"], new["methods"]) + old["properties"] = old.get("properties", []) + new.get("properties", []) + return old + +def classes_merger(key, old, new): + o = dict((v["name"], v) for v in old) + for v in new: + if o.has_key(v["name"]): + o[v["name"]] = class_merger(o[v["name"]], v) + else: + o[v["name"]] = v + return list(o.values()) + mergers = { + "extension": (extension_info_merger, []), + "domains": (domains_merger, []), + "constants": (constants_merger, []), + "classes": (classes_merger, []), } def merge_load_specs(filenames): -- cgit v1.2.1 From ae287dd3db7f7bb1db50b93c224073933c7312b9 Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Mon, 19 Oct 2009 13:36:42 +0100 Subject: Move demo extension from rabbitmq-server to rabbitmq-codegen; start documentation --- README.extensions.md | 4 ++++ demo_extension.json | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 README.extensions.md create mode 100644 demo_extension.json diff --git a/README.extensions.md b/README.extensions.md new file mode 100644 index 0000000..5b4b2c2 --- /dev/null +++ b/README.extensions.md @@ -0,0 +1,4 @@ +# Protocol extensions + +((TODO: shapes of the main document; shapes of extension documents; +how to invoke the compiler; merge behaviour)) diff --git a/demo_extension.json b/demo_extension.json new file mode 100644 index 0000000..4a65dbe --- /dev/null +++ b/demo_extension.json @@ -0,0 +1,18 @@ +{ + "extension": { + "name": "demo", + "version": "1.0", + "copyright": "Copyright (C) 2009 LShift Ltd." + }, + "domains": [ + ["foo-domain", "shortstr"] + ], + "constants": [ + {"name": "FOO-CONSTANT", "value": 121212} + ], + "classes": [ + {"name": "demo", + "id": 555, + "methods": [{"name": "one", "id": 1, "arguments": []}]} + ] +} \ No newline at end of file -- cgit v1.2.1 From becc3b0b7580c0c08105ec208ea590dc27972bce Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Sun, 23 May 2010 22:55:52 +1200 Subject: Check for duplicate properties. --- amqp_codegen.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/amqp_codegen.py b/amqp_codegen.py index 1ade01f..2fcf9b6 100644 --- a/amqp_codegen.py +++ b/amqp_codegen.py @@ -64,7 +64,7 @@ class AmqpSpecFileMergeConflict(Exception): pass def default_spec_value_merger(key, old, new): if old is None or old == new: return new - raise AmqpSpecFileMergeConflict((key, old, new)) + raise AmqpSpecFileMergeConflict(key, old, new) def extension_info_merger(key, old, new): return old + [new] @@ -89,13 +89,23 @@ def methods_merger(classname, old, new): o = dict((v["name"], v) for v in old) for v in new: if o.has_key(v["name"]): - raise AmqpSpecFileMergeConflict(("class", classname), old, new) + raise AmqpSpecFileMergeConflict(("class-methods", classname), old, new) o[v["name"]] = v return list(o.values()) +def properties_merger(classname, old, new): + oldnames = set(v["name"] for v in old) + newnames = set(v["name"] for v in new) + clashes = oldnames.intersection(newnames) + if clashes: + raise AmqpSpecFileMergeConflict(("class-properties", classname), old, new) + return old + new + def class_merger(old, new): old["methods"] = methods_merger(old["name"], old["methods"], new["methods"]) - old["properties"] = old.get("properties", []) + new.get("properties", []) + old["properties"] = properties_merger(old["name"], + old.get("properties", []), + new.get("properties", [])) return old def classes_merger(key, old, new): -- cgit v1.2.1 From aa784084953c938fe5c85f8d8a99d7afd28a44d2 Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Sun, 23 May 2010 22:56:49 +1200 Subject: Documentation. --- README.extensions.md | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 2 deletions(-) diff --git a/README.extensions.md b/README.extensions.md index 5b4b2c2..33f0301 100644 --- a/README.extensions.md +++ b/README.extensions.md @@ -1,4 +1,162 @@ # Protocol extensions -((TODO: shapes of the main document; shapes of extension documents; -how to invoke the compiler; merge behaviour)) +The `amqp_codegen.py` AMQP specification compiler has recently been +enhanced to take more than a single specification file, which allows +AMQP library authors to include extensions to the core protocol +without needing to modify the core AMQP specification file as +distributed. + +The compiler is invoked with the path to a single "main" specification +document and zero or more paths to "extension" documents. + +The order of the extensions matters: any later class property +definitions, for instance, are added to the list of definitions in +order of appearance. In general, composition of extensions with a core +specification document is therefore non-commutative. + +## The main document + +Written in the style of a +[json-shapes](http://github.com/tonyg/json-shapes) schema: + + DomainDefinition = _and(array_of(string()), array_length_equals(2)); + + ConstantDefinition = { + "name": string(), + "value": anything(), + "class": optional(_or("soft-error", "hard-error")) + }; + + FieldDefinition = { + "name": string(), + "type": string(), + "default-value": optional(anything()) + }; + + MethodDefinition = { + "name": string(), + "id": number(), + "arguments": array_of(FieldDefinition), + "synchronous": optional(boolean()) + }; + + ClassDefinition = { + "name": string(), + "id": number(), + "methods": array_of(MethodDefinition), + "properties": optional(array_of(FieldDefinition)) + }; + + MainDocument = { + "major-version": number(), + "minor-version": number(), + "revision": optional(number()), + "port": number(), + "domains": array_of(DomainDefinition), + "constants": array_of(ConstantDefinition), + "classes": array_of(ClassDefinition), + } + +Within a `FieldDefinition`, the keyword `domain` can be used instead +of `type`, but `type` is preferred and `domain` is deprecated. + +## Extensions + +Written in the style of a +[json-shapes](http://github.com/tonyg/json-shapes) schema, and +referencing some of the type definitions given above: + + ExtensionDocument = { + "extension": anything(), + "domains": array_of(DomainDefinition), + "constants": array_of(ConstantDefinition), + "classes": array_of(ClassDefinition) + }; + +The `extension` keyword is used to describe the extension informally +for human readers. Typically it will be a dictionary, with members +such as: + + { + "name": "The name of the extension", + "version": "1.0", + "copyright": "Copyright (C) 1234 Yoyodyne, Inc." + } + +## Merge behaviour + +In the case of conflicts between values specified in the main document +and in any extension documents, type-specific merge operators are +invoked. + + - Any doubly-defined domain names are regarded as true + conflicts. Otherwise, all the domain definitions from all the main + and extension documents supplied to the compiler are merged into a + single dictionary. + + - Constant definitions are treated as per domain names above, + *mutatis mutandis*. + + - Classes and their methods are a little trickier: if an extension + defines a class with the same name as one previously defined, then + only the `methods` and `properties` fields of the extension's class + definition are attended to. + + - Any doubly-defined method names or property names within a class + are treated as true conflicts. + + - Properties defined in an extension are added to the end of the + extant property list for the class. + + (Extensions are of course permitted to define brand new classes as + well as to extend existing ones.) + + - Any other kind of conflict leads to a raised + `AmqpSpecFileMergeConflict` exception. + +## Invoking the spec compiler + +Your code generation code should invoke `amqp_codegen.do_main` with +two functions as arguments: one for generating "header-file" text, and +one for generating "implementation-file" text. The `do_main` function +will parse the command-line arguments supplied when python was +invoked. + +The command-line will be parsed as: + + python your_codegen.py [ ...] + +where `` is either the word `header` or the word `body`, +to select which generation function is called by `do_main`. The +`` and `` arguments are file names of specification +documents containing expressions in the syntax given above. The +*final* argument on the command line, ``, is the name of the +source-code file to generate. + +Here's a tiny example of the layout of a code generation module that +uses `amqp_codegen`: + + import amqp_codegen + + def generateHeader(specPath): + spec = amqp_codegen.AmqpSpec(specPath) + ... + + def generateImpl(specPath): + spec = amqp_codegen.AmqpSpec(specPath) + ... + + if __name__ == "__main__": + amqp_codegen.do_main(generateHeader, generateImpl) + +The reasons for this split, such as they are, are that + + - many languages have separate "header"-type files (C and Erlang, to + name two) + - `Makefile`s often require separate rules for generating the two + kinds of file, but it's convenient to keep the generation code + together in a single python module + +The main reason things are laid out this way, however, is simply that +it's an accident of the history of the code. We may change the API to +`amqp_codegen` in future to clean things up a little. -- cgit v1.2.1 From 19891d34f4f9a4adcde5b7516c9dfd181228b930 Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Sun, 23 May 2010 23:11:06 +1200 Subject: More documentation. --- README.extensions.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/README.extensions.md b/README.extensions.md index 33f0301..3a067ba 100644 --- a/README.extensions.md +++ b/README.extensions.md @@ -23,7 +23,7 @@ Written in the style of a ConstantDefinition = { "name": string(), - "value": anything(), + "value": number(), "class": optional(_or("soft-error", "hard-error")) }; @@ -37,7 +37,8 @@ Written in the style of a "name": string(), "id": number(), "arguments": array_of(FieldDefinition), - "synchronous": optional(boolean()) + "synchronous": optional(boolean()), + "content": optional(boolean()) }; ClassDefinition = { @@ -60,6 +61,31 @@ Written in the style of a Within a `FieldDefinition`, the keyword `domain` can be used instead of `type`, but `type` is preferred and `domain` is deprecated. +Type names can either be a defined `domain` name or a built-in name +from the following list: + + - octet + - shortstr + - longstr + - short + - long + - longlong + - bit + - table + - timestamp + +Method and class IDs must be integers between 0 and 65535, +inclusive. Note that there is no specific subset of the space reserved +for experimental or site-local extensions, so be careful not to +conflict with IDs used by the AMQP core specification. + +If the `synchronous` field of a `MethodDefinition` is missing, it is +assumed to be `false`; the same applies to the `content` field. + +A `ConstantDefinition` with a `class` attribute is considered to be an +error-code definition; otherwise, it is considered to be a +straightforward numeric constant. + ## Extensions Written in the style of a -- cgit v1.2.1