Merge remote-tracking branch 'origin/master' into baserock/danielsilverstone/upgrade-gitano-fix-rsyncbaserock/danielsilverstone/upgrade-gitano-fix-rsync

18 files changed, 309 insertions, 7 deletions

diff --git a/.gitignore b/.gitignore
index 6a58e30..da8ed65 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 luacov.stats.out
 luacov.report.out
 
+html
diff --git a/Makefile b/Makefile
index f585d7e..6d733de 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,11 @@
-all: test
+all: test doc
 
 MODULES := lace lace.lex lace.compiler lace.builtin lace.engine lace.error
 LUA_VER := 5.1
 
-INST_BASE := /usr/local
+PREFIX ?= /usr/local
+
+INST_BASE := $(PREFIX)
 INST_ROOT := $(DESTDIR)$(INST_BASE)/share/lua/$(LUA_VER)
 
 MOD_FILES := $(patsubst %,%.lua,$(subst .,/,$(MODULES)))
@@ -18,6 +20,7 @@ LUA := LUA_PATH="$(shell pwd)/lib/?.lua;$(shell pwd)/extras/luacov/src/?.lua;;"
 
 clean:
 	$(RM) luacov.report.out luacov.stats.out
+	$(RM) -r html
 
 distclean: clean
 	find . -name "*~" -delete
@@ -41,3 +44,7 @@ test:
 .PHONY: interactive
 interactive:
 	$(LUA) -e'lace=require"lace"' -i
+
+doc:
+	@echo "Running LDoc"
+	@ldoc .
diff --git a/config.ld b/config.ld
new file mode 100644
index 0000000..e45aa68
--- /dev/null
+++ b/config.ld
@@ -0,0 +1,6 @@
+file = {"lib"}
+readme = "readme.md"
+format = "lua-markdown"
+project = "Lua Access Control Engine"
+title = "Lace - Lua Access Control Engine"
+dir = "html"
diff --git a/lib/lace.lua b/lib/lace.lua
index 2002cf3..2c76cb8 100644
--- a/lib/lace.lua
+++ b/lib/lace.lua
@@ -7,6 +7,16 @@
 -- For licence terms, see COPYING
 --
 
+--- Lua Access Control Engine.
+--
+-- The Lua Access Control Engine library consists primarily of a ruleset
+-- compiler and execution engine.  In addition there is a lexer and also a
+-- complex error reporting system designed to ensure application authors who
+-- use Lace in their projects can provide good error messages to their users.
+--
+-- * For compiling rulesets, see `lace.compiler.compile`.
+-- * For running compiled rulesets, see `lace.engine.run`.
+
 local lex = require "lace.lex"
 local compiler = require "lace.compiler"
 local builtin = require "lace.builtin"
diff --git a/lib/lace/builtin.lua b/lib/lace/builtin.lua
index 2f03549..147da6a 100644
--- a/lib/lace/builtin.lua
+++ b/lib/lace/builtin.lua
@@ -7,6 +7,14 @@
 -- For Licence terms, see COPYING
 --
 
+--- Lace builtin commands and match types.
+--
+-- The builtin match types and commands provided by Lace.  These commands and
+-- match types are supported automatically by all lace compiles.  The builtin
+-- command `default` and the builtin commands `allow` and `deny` collude with
+-- the compiler to ensure that all compiled rulesets will always either
+-- explicitly allow or deny access.
+
 local builtin = {}
 
 local engine = require "lace.engine"
@@ -53,12 +61,36 @@ end
 
 local unconditional_result, last_result
 
+--- Internal function to get/set the last result for unconditional access.
+--
+-- The last result (unconditional only) is stored so that defaults can be
+-- processed in the absence of a `default` statement.
+--
+-- This function exists to collude with `lace.compiler.internal_compile` so
+-- that it can synthesise default access statements if needed.
+--
+-- @tparam string|nil newv The new value for the last access result.
+--                         It should be one of `allow`, `deny` or a _nil_.
+-- @treturn string|nil The old (current) value for the last access result.
+-- @function get_set_last_unconditional_result
 local function get_set_last_unconditional_result(newv)
    local ret = unconditional_result
    unconditional_result = newv
    return ret
 end
 
+--- Internal function to get/set the last result for access.
+--
+-- The last result (conditional perhaps) is stored so that defaults can be
+-- processed in the absence of a `default` statement.
+--
+-- This function exists to collude with `lace.compiler.internal_compile` so
+-- that it can synthesise default access statements if needed.
+--
+-- @tparam string|nil newv The new value for the last access result.
+--                         It should be one of `allow`, `deny` or a _nil_.
+-- @treturn string|nil The old (current) value for the last access result.
+-- @function get_set_last_result
 local function get_set_last_result(newv)
    local ret = last_result
    last_result = newv
@@ -78,6 +110,21 @@ local function _do_return(exec_context, result, reason, cond)
    return result, reason
 end
 
+--- Compile an `allow` or `deny`.
+--
+-- (_Note: this is also `commands.deny`_)
+--
+-- Allowing and denying access is, after all, what access control lists are all
+-- about.  This function compiles in an `allow` or `deny` statement including
+-- noting what kind of access statement it is and what 
+--
+-- @tparam table compcontext The compilation context
+-- @tparam string result The result to be compiled (`allow` or `deny`).
+-- @tparam string reason The reason to be returned to the user for this.
+-- @tparam[opt] string ... The conditions placed on this `allow` or `deny`.
+-- @treturn table The compiled `allow`/`deny`.
+-- @function commands.allow
+-- @alias commands.deny
 local function _return(compcontext, result, reason, ...)
    if result ~= "allow" and result ~= "deny" then
       return err.error("Unknown result: " .. result, {1})
@@ -103,6 +150,24 @@ builtin.deny = _return
 
 --[ Default for Allow and Deny ]------------------------------------
 
+--- Compile a `default` command.
+--
+-- All rulesets must, ultimately, allow or deny access.  The `default` command
+-- allows rulesets to define whether they are permissive (defaulting to
+-- `allow`) or proscriptive (defaulting to `deny`).
+--
+-- In addition, setting default causes a record to be made, preventing
+-- additional attempts to set a default access mode.  This ensures that once
+-- the default has been selected, additional ruleset included (perhaps from
+-- untrusted sources) cannot change the default behaviour.
+--
+-- @tparam table compcontext The compilation context
+-- @tparam string def The command which triggered this compilation. (`default`)
+-- @tparam string result The default result (`allow` or `deny`)
+-- @tparam string reason The reason to be given.
+-- @tparam[opt] * unwanted If _unwanted_ is anything but nil, an error occurs.
+-- @treturn table A null command
+-- @function commands.default
 function builtin.default(compcontext, def, result, reason, unwanted)
    assert(def == "default", "Somehow, builtin.default got something odd")
    if type(result) ~= "string" then
@@ -164,6 +229,24 @@ local function _controlfn(ctx, name)
    return cfn
 end
 
+--- Compile a definition command
+--
+-- Definitions are a core behaviour of Lace.  This builtin allows the ruleset
+-- to define additional conditions on which `allow`, `deny` and `include` can
+-- operate.  
+--
+-- @tparam table compcontext The compilation context.
+-- @tparam string define The word which triggered this compilation command.
+--                       (`define`)
+-- @tparam string name The name being defined.
+-- @tparam string controltype The control type to be used. (Such as `anyof`,
+--                            `allof` or any of the match types defined by
+--                            the caller of the compiler).
+-- @tparam[opt] string ... The content of the definition (consumed by the
+--                         match type compiler).
+-- @treturn table The compiled definition command.
+-- @function commands.define
+-- @alias commands.def
 function builtin.define(compcontext, define, name, controltype, ...)
    if type(name) ~= "string" then
       return err.error("Expected name, got nothing", {1})
@@ -217,6 +300,27 @@ local function _do_include(exec_context, ruleset, conds)
    return result, msg
 end
 
+--- Compile an `include` command.
+--
+-- Compile a lace `include` command.  This uses the exported internal loader
+-- function `lace.compiler.internal_loader` to find a loader and if it finds
+-- one, it uses the internal compilation function
+-- `lace.compiler.internal_compile` to compile the loaded source before
+-- constructing a runtime "inclusion" which deals with the conditions before
+-- running the sub-ruleset if appropriate.
+--
+-- Regardless of the conditions placed on the include statement, includes are
+-- always processed during compilation.
+--
+-- @tparam table comp_context The compilation context
+-- @tparam string cmd The command which triggered this include command.
+--                    (`include` or `include?`)
+-- @tparam string file The file (source name) to include.
+-- @tparam[opt] string ... Zero or more conditions under which the included
+--                         content will be run by the engine.  If there are no
+--                         conditions then the include is unconditional.
+-- @treturn table The compiled inclusion command.
+-- @function commands.include
 function builtin.include(comp_context, cmd, file, ...)
    local safe_if_not_present = cmd:sub(-1) == "?"
 
diff --git a/lib/lace/compiler.lua b/lib/lace/compiler.lua
index 1a89897..26a4e23 100644
--- a/lib/lace/compiler.lua
+++ b/lib/lace/compiler.lua
@@ -7,6 +7,12 @@
 -- For licence terms, see COPYING
 --
 
+--- The compiler for the Lua Access Control engine.
+--
+-- Lace works hard to give you good error message information when encountering
+-- problems with your rulesets.  The compiler gathers a lot of debug
+-- information and stores it alongside the compiled ruleset.
+
 local lex = require "lace.lex"
 local builtin = require "lace.builtin"
 local err = require "lace.error"
@@ -19,6 +25,15 @@ local function _fake_command(ctx)
    return err.error("Command is disabled by context")
 end
 
+--- Internal loader abstraction.
+--
+-- Used by `lace.builtin.commands.include`, this function returns a loader
+-- which can be used to acquire more content during compilation of a Lace
+-- ruleset.
+--
+-- @tparam table ctx The Lace compiliation context
+-- @treturn function A loader function
+-- @function internal_loader
 local function _loader(ctx)
    -- We know the context is a table with a _lace table, so retrieve
    -- the loader function.  If it's absent, return a loader which
@@ -61,6 +76,18 @@ local function compile_one_line(compcontext, line)
    return cmdfn(compcontext, unpack(args))
 end
 
+--- Internal ruleset compilation.
+--
+-- Internal ruleset compilation function.  This function should not be used
+-- except from compilation commands such as `lace.builtin.commands.include`.
+-- This function is much less forgiving of issues than `lace.compiler.compile`.
+--
+-- @tparam table compcontext Compilation context
+-- @tparam string sourcename Source name
+-- @tparam string content Source content
+-- @tparam boolean suppress_default Suppress the use of a default rule.
+-- @treturn table Compiled Lace ruleset
+-- @function internal_compile
 local function internal_compile_ruleset(compcontext, sourcename, content, suppress_default)
    assert(type(compcontext) == "table", "Compilation context must be a table")
    assert(type(compcontext._lace) == "table", "Compilation context must contain a _lace table")
@@ -144,6 +171,19 @@ local function internal_compile_ruleset(compcontext, sourcename, content, suppre
    return ruleset
 end
 
+--- Compile a lace ruleset.
+--
+-- Compile a lace ruleset so that it can be executed by `lace.engine.run`
+-- later.  If you provide content then it is compiled using the source name as
+-- the name used in error messages.  If you do not supply any content then Lace
+-- will construct an implicit include of the given source name.
+--
+-- @tparam table ctx Compilation context
+-- @tparam string src Source name
+-- @tparam ?string cnt Source contents (nil to cause an implicit
+--                                      include of _src_)
+-- @treturn table Compiled Lace ruleset
+-- @function compile
 local function compile_ruleset(ctx, src, cnt)
    -- Augment the compiler context with a false
    -- source so that we can be sure the expect early errors to stand a chance
diff --git a/lib/lace/engine.lua b/lib/lace/engine.lua
index 5484c9b..45c2c33 100644
--- a/lib/lace/engine.lua
+++ b/lib/lace/engine.lua
@@ -7,6 +7,12 @@
 -- For licence terms, see COPYING
 --
 
+--- The runtime engine for the Lua Access Control Engine
+--
+-- Once a ruleset has been compiled, it can be run for multiple inputs without
+-- needing to be recompiled.  This is handy for controlling access to a
+-- long-lived daemon such as an HTTP proxy.
+
 local err = require 'lace.error'
 
 local function _dlace(ctx)
@@ -15,6 +21,16 @@ local function _dlace(ctx)
    return ret
 end
 
+
+--- Set a definition.
+--
+-- @tparam table exec_context The execution context for the runtime.
+-- @tparam string name The name of the define to set.
+-- @tparam table defn The definition function to use.
+-- @treturn boolean Returns true if the definition was set successfully.
+-- @treturn nil|table If the definition was not set successfully then this is
+--                    the error table ready to have context added to it.
+-- @function define
 local function set_define(exec_context, name, defn)
    local dlace = _dlace(exec_context)
    dlace.defs = dlace.defs or {}
@@ -25,6 +41,17 @@ local function set_define(exec_context, name, defn)
    return true
 end
 
+--- Test a definition.
+--
+-- @tparam table exec_context The execution context for the runtime.
+-- @tparam string name The name of the define to test.
+-- @treturn boolean|nil If the named definition does not exist, this is nil.
+--                      Otherwise it is true iff. the definition's function
+--                      results in true.
+-- @treturn nil|table If the named definition does not exist, this is the error
+--                    table ready for filling out with more context.
+--                    Otherwise it is nil.
+-- @function test
 local function test_define(exec_context, name)
    local dlace = _dlace(exec_context)
    dlace.defs = dlace.defs or {}
@@ -36,6 +63,22 @@ local function test_define(exec_context, name)
    return defn.fn(exec_context, unpack(defn.args))
 end
 
+--- Internal routine for running sub-rulesets
+--
+-- @tparam table ruleset The compiled ruleset to run.
+-- @tparam table exec_context The execution context for the runtime.
+-- @treturn nil|boolean|string The first return value is `nil` in the case
+--                             of a runtime error, `false` if a Lace error
+--                             was encountered during runtime, otherwise it it
+--                             a result string (typically `allow` or `deny`).
+--                             In addition, internally, an empty result string
+--                             will be returned if no result was set by the
+--                             sub-ruleset.
+-- @treturn nil|string If an error was encountered, this is the error message,
+--                     otherwise it is an additional message to go with the
+--                     result if there was one, or `nil` in the case of no
+--                     result value being set by the ruleset.
+-- @function internal_run
 local function internal_run_ruleset(ruleset, exec_context)
    -- We iterate the ruleset, returning the first time
    -- a rule either errors, or returns a stopping result
@@ -58,6 +101,17 @@ local function internal_run_ruleset(ruleset, exec_context)
    return ""
 end
 
+--- Run a ruleset.
+--
+-- @tparam table ruleset The compiled ruleset to run.
+-- @tparam table exec_context The execution context for the runtime.
+-- @treturn nil|boolean|string The first return value is `nil` in the case
+--                             of a runtime error, `false` if a Lace error
+--                             was encountered during runtime, otherwise it it
+--                             a result string (typically `allow` or `deny`).
+-- @treturn string If an error was encountered, this is the error message,
+--                 otherwise it is an additional message to go with the result.
+-- @function run
 local function run_ruleset(ruleset, exec_context)
    local ok, ret, msg = xpcall(function() 
 				  return internal_run_ruleset(ruleset, exec_context)
diff --git a/lib/lace/error.lua b/lib/lace/error.lua
index 29683e4..96998bd 100644
--- a/lib/lace/error.lua
+++ b/lib/lace/error.lua
@@ -7,6 +7,25 @@
 -- For licence terms, see COPYING
 --
 
+--- Error routines for Lace ruleset compilers and runtime engines.
+--
+-- Errors are a critical part of anything user-facing.  Lace works very
+-- hard to ensure that it can report errors effectively so that the author
+-- of the ruleset (the user of the application which is using Lace) can work
+-- out what went wrong and how to fix it.
+
+--- Report an error.
+--
+-- Report an error, including indicating which words caused the problem.
+-- The words are 1-indexed from the start of whatever routine is trying
+-- to consume words.
+--
+-- @tparam string str The error message.
+-- @tparam {number,...}|nil words A list of the words causing this error.
+-- @tparam boolean rnil Whether to return nil (indicating a programming error).
+-- @treturn boolean|nil,table The compilation result (false or nil) and a
+--                            compilation error table
+-- @function error
 local function _error(str, words, rnil)
    local ret = false
    if rnil then
@@ -15,6 +34,16 @@ local function _error(str, words, rnil)
    return ret, { msg = str, words = words or {} }
 end
 
+--- Offset an error's recorded wordset.
+--
+-- Since errors carry word indices, if the layers of the compiler or runtime
+-- alter the offsets, this routine can be used to offset the word indices
+-- in an error message.
+--
+-- @tparam table err The error table
+-- @tparam number offs The offset by which to adjust the error words.
+-- @treturn table The error table (mutated by the offset).
+-- @function offset
 local function _offset(err, offs)
    if not err.words then
       err.words = {}
@@ -25,12 +54,35 @@ local function _offset(err, offs)
    return err
 end
 
+--- Augment an error with source information
+--
+-- In order for errors to be useful they need to be augmented with the source
+-- document in which they occurred and the line number on which they occurred.
+-- This function allows the compiler (or runtime) to do just that.
+--
+-- @tparam table err The error table to augment
+-- @tparam table source The lexically analysed source document.
+-- @tparam linenr number The line number on which the error occurred.
+-- @treturn table The error table (mutated with the source information).
+-- @function augment
 local function _augment(err, source, linenr)
    err.source = source
    err.linenr = linenr
    return err
 end
 
+--- Render an error down to a string.
+--
+-- Error tables carry a message, an optional set of words which caused the
+-- error (if known) and a lexically analysed source and line number.
+--
+-- This function renders that information down to a multiline string which can
+-- usefully be presented to the user of an application using Lace for access
+-- control.
+--
+-- @tparam table err The error table.
+-- @treturn string A multiline string rendering of the error.
+-- @function render
 local function _render(err)
    -- A rendered error has four lines
    -- The first line is the error message
diff --git a/lib/lace/lex.lua b/lib/lace/lex.lua
index d9434b1..9cc2a51 100644
--- a/lib/lace/lex.lua
+++ b/lib/lace/lex.lua
@@ -7,7 +7,15 @@
 -- For Licence terms, see COPYING
 --
 
-local function lex_one_line(line)
+--- Lace Internals - Ruleset lexer.
+--
+-- The lexer for Lace is only used internally and is generally not accessed
+-- from outside of Lace.  It is exposed only for testing and validation
+-- purposes.
+
+local M = {}
+
+local function _lex_one_line(line)
    local r = {}
    local acc = ""
    local c
@@ -83,7 +91,27 @@ local function lex_one_line(line)
    return r, warnings
 end
 
-local function lex_a_ruleset(ruleset, sourcename)
+local lexer_line_cache = {}
+
+local function lex_one_line(line)
+   if not lexer_line_cache[line] then
+      lexer_line_cache[line] = { _lex_one_line(line) }
+   end
+   return lexer_line_cache[line][1], lexer_line_cache[line][2]
+end
+
+local cached_full_lexes = {}
+
+--- Lexically analyse a ruleset.
+-- @tparam string ruleset The ruleset to lex.
+-- @tparam string sourcename The name of the source to go into debug info.
+-- @treturn table A list of lexed lines, each line being a table of tokens
+--                with their associated debug information.
+function M.string(ruleset, sourcename)
+   if cached_full_lexes[sourcename] and
+      cached_full_lexes[sourcename][ruleset] then
+      return cached_full_lexes[sourcename][ruleset]
+   end
    local lines = {}
    local ret = { source = sourcename, lines = lines }
    local n = 1
@@ -109,9 +137,9 @@ local function lex_a_ruleset(ruleset, sourcename)
       lines[n] = linetab
       n = n + 1
    end
+   cached_full_lexes[sourcename] = cached_full_lexes[sourcename] or {}
+   cached_full_lexes[sourcename][ruleset] = ret
    return ret
 end
 
-return {
-   string = lex_a_ruleset,
-}
+return M
diff --git a/doc/compilation.mdwn b/old-docs/compilation.mdwn
index 37876db..37876db 100644
--- a/doc/compilation.mdwn
+++ b/old-docs/compilation.mdwn
diff --git a/doc/developing.mdwn b/old-docs/developing.mdwn
index f82a593..f82a593 100644
--- a/doc/developing.mdwn
+++ b/old-docs/developing.mdwn
diff --git a/doc/execution.mdwn b/old-docs/execution.mdwn
index 50049a9..50049a9 100644
--- a/doc/execution.mdwn
+++ b/old-docs/execution.mdwn
diff --git a/doc/index.mdwn b/old-docs/index.mdwn
index 2b7f485..2b7f485 100644
--- a/doc/index.mdwn
+++ b/old-docs/index.mdwn
diff --git a/doc/syntax-allow-deny.mdwn b/old-docs/syntax-allow-deny.mdwn
index c227abf..c227abf 100644
--- a/doc/syntax-allow-deny.mdwn
+++ b/old-docs/syntax-allow-deny.mdwn
diff --git a/doc/syntax-default.mdwn b/old-docs/syntax-default.mdwn
index 2304ed5..2304ed5 100644
--- a/doc/syntax-default.mdwn
+++ b/old-docs/syntax-default.mdwn
diff --git a/doc/syntax-define.mdwn b/old-docs/syntax-define.mdwn
index 579fe44..579fe44 100644
--- a/doc/syntax-define.mdwn
+++ b/old-docs/syntax-define.mdwn
diff --git a/doc/syntax-include.mdwn b/old-docs/syntax-include.mdwn
index a95f489..a95f489 100644
--- a/doc/syntax-include.mdwn
+++ b/old-docs/syntax-include.mdwn
diff --git a/doc/syntax.mdwn b/old-docs/syntax.mdwn
index 75a2b1e..75a2b1e 100644
--- a/doc/syntax.mdwn
+++ b/old-docs/syntax.mdwn