-- gitano.repository -- -- Create/destroy/blahblah repositories under Gitano management -- -- Copyright 2012 Daniel Silverstone -- local gall = require 'gall' local luxio = require 'luxio' local sio = require 'luxio.simple' local sp = require 'luxio.subprocess' local log = require 'gitano.log' local config = require 'gitano.config' local util = require 'gitano.util' local lace = require 'gitano.lace' local markdown = require 'gitano.markdown' local clod = require 'clod' local base_rules = [[ -- Empty Ruleset ]] local base_config = [[ project.head "refs/heads/master" project.description "" project.owner "gitano/nobody" ]] local admin_name = { realname = "Gitano", email = "gitano@gitano-admin.git" } local adminrefname = "refs/gitano/admin" local repo_method = {} function repo_method:load_from_admin_ref(filename) local admincommit = self.git:get(adminrefname) local admintree = admincommit.content.tree.content local flat_tree = gall.tree.flatten(admintree) local entry = flat_tree[filename] if not entry then return nil, "Not found: " .. admincommit.sha .. "::" .. filename end if entry.obj.type ~= "blob" then return nil, admincommit.sha .. "::" .. filename .. ": Not a blob" end return entry.obj.content, admincommit.sha .. "::" .. filename end function repo_method:uses_hook(hookname) -- If the hook is global, then we use it if config.has_global_hook(self.config, hookname) then return true end local admincommit = self.git:get(adminrefname) local admintree = admincommit.content.tree.content local flat_tree = gall.tree.flatten(admintree) local entry = flat_tree["hooks/" .. hookname .. ".lua"] if not entry then return false end if entry.type ~= "blob" then return false end -- Got a hook, it's a blob, sounds good return true end function repo_method:fs_path() return ("%s/%s.git"):format(config.repo_path(), self.name) end function repo_method:run_checks() -- Things we check for are: -- + appropriate gitano/* refs -- + hooks are all in place if not self.git:get_ref(adminrefname) then -- Admin branch is missing, create one. -- First, the blob for the rules local rules, msg = gall.object.create(self.git, "blob", base_rules) if not rules then return nil, msg end local conf, msg = gall.object.create(self.git, "blob", base_config) if not conf then return nil, msg end -- Next the tree containing the blob local flat_tree = { ["rules/main.lace"] = rules, ["project.conf"] = conf, } local tree, msg = gall.tree.create(self.git, flat_tree) if not tree then return nil, msg end -- Now a commit of the blob, using the gitano admin identity local commit, msg = gall.commit.create(self.git, { author = admin_name, committer = admin_name, message = "Initial admin tree", tree = tree }) if not commit then return nil, msg end -- Finally create/update the gitano/admin ref local ok, msg = self.git:update_ref(adminrefname, commit.sha) if not ok then return nil, msg end end -- Admin tree exists, validate and load it... local ok, tab = self:validate_admin_sha(self.git:get_ref(adminrefname)) if not ok then return nil, tab end -- Validated, copy the data from tab across for k, v in pairs(tab) do self[k] = v end -- From here on down, self:conf_*() will work -- Remove trailing newlines from project descriptions local desc = self:conf_get "project.description" if desc:match("\n$") then desc = desc:match("^(.-)\n+$") self:conf_set("project.description", desc) end -- Now check and update the hooks local ok, msg = self:check_and_upgrade_hooks() if not ok then return nil, msg end -- Now update the git-local files from the rules and metadata local ok, msg = self:check_local_git_files() if not ok then return nil, msg end -- And all is well return true end function repo_method:conf_get(key) return self.project_config.settings[key] end function repo_method:conf_set(key, value) self.project_config.settings[key] = value end function repo_method:run_lace(context) self:populate_context(context) config.populate_context(self.config, context) log.ddebug("Running lace for <" .. self.name .. ">, operation <" .. (context.operation or "UNKNOWN") .. ">") return lace.run(self.lace, context) end function repo_method:check_local_git_files() -- Construct a fake read operation for anonymous -- accept => git-daemon-export-ok local context = { ["operation"] = "read", ["source"] = "git", ["user"] = "gitano/anonymous", } local action, reason = self:run_lace(context) local anonexport = false if action == "allow" then anonexport = true end if anonexport then log.ddebug("<" .. self.name .. ">: Anonymous read allowed") local fh = sio.open(self:fs_path() .. "/git-daemon-export-ok", "wc") fh:close() else log.ddebug("<" .. self.name .. ">: Anonymous read not allowed") luxio.unlink(self:fs_path() .. "/git-daemon-export-ok") end -- Now store our description into the description file local fh, err = sio.open(self:fs_path() .. "/description.new", "wc") if fh then fh:write(self:conf_get "project.description") fh:write("\n") fh:close() local ok, errno = luxio.rename(self:fs_path() .. "/description.new", self:fs_path() .. "/description") if ok ~= 0 then log.warn("Unable to rename description into place:", luxio.strerror(errno)) end else log.warn("Unable to write description:", tostring(err)) end -- And now our readme if we have one if self.readme_html then local fh, err = sio.open(self:fs_path() .. "/README.html.new", "wc") if fh then fh:write(self.readme_html) fh:close() local ok, errno = luxio.rename(self:fs_path() .. "/README.html.new", self:fs_path() .. "/README.html") if ok ~= 0 then log.warn("Unable to rename README.html into place:", luxio.strerror(errno)) end else log.warn("Unable to write README.html out:", tostring(err)) end else -- Remove any readme (and ignore errors) luxio.unlink(self:fs_path() .. "/README.html") end -- Check that our HEAD is correctly configured local our_head = self:conf_get "project.head" if self.git.HEAD ~= our_head then self.git:symbolic_ref("HEAD", our_head) self.git.HEAD = our_head end -- Check that our owner is set correctly local owner = self.config.users[self:conf_get "project.owner"] if owner then self.git:config("gitweb.owner", owner.real_name .. " <" .. owner.email_address .. ">") end -- Construct a cgitrc in case we're using cgit instead of gitweb local cgitrc = {} local function add_cgitrc(key, value) cgitrc[#cgitrc+1] = ("%s=%s"):format(key, tostring(value)) end for _, suffix in ipairs { "readme", "section", "logo", "logo-link", "defbranch", "clone-url", "snapshots" } do local v = self.project_config.settings["cgitrc." .. suffix] if v then add_cgitrc(suffix, v) end end if owner then add_cgitrc("owner", owner.real_name .. " <" .. owner.email_address .. ">") end local description = self:conf_get "project.description" if description and description ~= "" then add_cgitrc("desc", description) end table.sort(cgitrc) cgitrc[#cgitrc+1] = "" cgitrc = table.concat(cgitrc, "\n") local fh, err = sio.open(self:fs_path() .. "/cgitrc.new", "wc") if fh then fh:write(cgitrc) fh:close() local ok, errno = luxio.rename(self:fs_path() .. "/cgitrc.new", self:fs_path() .. "/cgitrc") if ok ~= 0 then log.warn("Unable to rename cgitrc into place:", luxio.strerror(errno)) end else log.warn("Unable to write cgitrc out:", tostring(err)) end return true end function repo_method:check_and_upgrade_hooks() log.ddebug("Checking hooks for", self.name) for _, hook in ipairs {"pre-receive", "update", "post-receive"} do local hookfile = ("%s/hooks/%s"):format(self:fs_path(), hook) local hooktarget = ("%s/gitano-%s-hook"):format(config.lib_bin_path(), hook) local ok, linkv = luxio.readlink(hookfile) local redo = false if ok > 0 then if linkv ~= hooktarget then redo = true end else redo = true end if redo then luxio.unlink(hookfile) luxio.symlink(hooktarget, hookfile) end end return true end function repo_method:validate_admin_sha(sha) if sha == string.rep("0", 40) then return nil, "A deleted admin ref cannot be loaded" end local commit = self.git:get(sha) local tree = gall.tree.flatten(commit.content.tree.content) local function is_blob(thingy) return thingy and thingy.type and thingy.type == "blob" end if not is_blob(tree["project.conf"]) then return nil, "Unable to find project.conf in repository" end local conf_text = tree["project.conf"].obj.content local conf, err = clod.parse(conf_text, self.name .. ":" .. sha .. ":project.conf") if not conf then return nil, err end local ret = { project_config = conf } -- Verify if the clod config needs "migration" if conf.settings["owner"] and not conf.settings["project.owner"] then conf.settings["project.owner"] = conf.settings["owner"] conf.settings["owner"] = nil end if conf.settings["description"] and not conf.settings["project.description"] then conf.settings["project.description"] = conf.settings["description"] conf.settings["description"] = nil end if conf.settings["head"] and not conf.settings["project.head"] then conf.settings["project.head"] = conf.settings["head"] conf.settings["head"] = nil end -- Now verify that the clod has sufficient values to know -- what we're doing if not conf.settings["project.owner"] then conf.settings["project.owner"] = "gitano/nobody" end if not conf.settings["project.description"] then conf.settings["project.description"] = "" end if not conf.settings["project.head"] then conf.settings["project.head"] = "refs/heads/master" end if is_blob(tree["README.mdwn"]) then local mdwn_src = tree["README.mdwn"].obj.content local ok, res = pcall(markdown.convert_to_html, mdwn_src) if ok then ret.readme_html = res ret.readme_mdwn = mdwn_src else log.warn("<" .. self.name .. "> Markdown:", res) end end -- Generate a Lace for the project at this ref local lace, msg = lace.compile(self, sha) if not lace then return nil, msg end -- And store it for use later ret.lace = lace return true, ret end function repo_method:populate_context(context) util.add_splitable(context, "repository", self.name, "/", "dirname", "basename") if self.is_nascent then context["owner"] = "gitano/nobody" else context["owner"] = self:conf_get "project.owner" end context["_repo"] = self end function repo_method:realise() if not self.is_nascent then return false, "Cannot realise a non-nascent repository." end if not util.mkdir_p(util.dirname(self:fs_path())) then return false, "Cannot prepare path leading to repository." end local r, msg = gall.repository.create(self:fs_path()) if not r then return false, msg end self.git = r self.is_nascent = nil -- Finally, we're not nascent, validate the repo return self:run_checks() end function repo_method:set_owner(newowner) local oldowner = self:conf_get "project.owner" self:conf_set("project.owner", newowner) local ok, msg = self:save_admin("Setting owner to " .. newowner) if not ok then self:conf_set("project.owner", oldowner) return nil, msg end log.state("<" .. self.name .. "> Set owner to <" .. newowner .. ">") return true end function repo_method:set_description(newdesc) local olddesc = self:conf_get "project.description" self:conf_set("project.description", newdesc) local ok, msg = self:save_admin("Changing description\n\n" .. newdesc) if not ok then self:conf_set("project.description", olddesc) return nil, msg end log.state("<" .. self.name .. "> Changed description") return true end function repo_method:set_readme(newreadme) local oldreadme = self.readme_mdwn self.readme_mdwn = newreadme local ok, msg = self:save_admin("Changing readme") if not ok then self.readme_mdwn = oldreadme return nil, msg end log.state("<" .. self.name .. "> Changed readme") return true end function repo_method:set_head(newhead) if not newhead:match("^refs/") then newhead = "refs/heads/" .. newhead end local oldhead = self:conf_get "project.head" self:conf_set("project.head", newhead) local ok, msg = self:save_admin() if not ok then self:conf_set("project.head", oldhead) return nil, msg end log.state("<" .. self.name .. "> Set head to <" .. newhead .. ">") return true end function repo_method:generate_confirmation(notes) -- Generate a confirmation token. -- To do this, we read *ALL* refs in the repository and their -- sha1 sums. We then hash all that to generate a token -- for this repository at its current state. local refs = self.git:all_refs() local refnames = {} for ref in pairs(refs) do refnames[#refnames+1] = ref end table.sort(refnames) local str = notes or "" log.debug("Calculating confirmation token for", self.name) for i = 1, #refnames do local ref, sha = refnames[i], refs[refnames[i]] log.ddebug(ref, "@", sha) str = str .. ref .. "@" .. sha .. "\n" end return self.git:hash_object("blob", str, false) or "ARGH!" end function repo_method:destroy_self(call_me) util.mkdir_p(config.repo_path() .. "/.graveyard") local graveyard_path = config.repo_path() .. "/.graveyard/".. call_me -- Is our destination location in the graveyard free? local ok, err = luxio.stat(graveyard_path) if ok == 0 then return false, "Our grave site is already occupied! Cannot destroy." end if err ~= luxio.ENOENT then return false, "Could not check grave site for occupancy. Cannot destroy." end -- Grave site is not occupied, attempt to rename ourselves local ok, err = luxio.rename(self:fs_path(), graveyard_path) if ok ~= 0 then return false, "Cannot destroy. rename() returned " .. luxio.strerror(err) end -- Successfully renamed ourselves, we're destroyed. return true end function repo_method:rename_to(somename) -- Same cleanup as in find... if somename:match(".%.git$") then somename = somename:match("^(.+)%.git$") end -- Remove any '.' somename = somename:gsub("%.", "") -- Remove any leading or trailing / somename = somename:match("^/*(.-)/*$") local newpath = self.fs_path({name=somename,config=self.config}) if not util.mkdir_p(util.dirname(newpath)) then return false, "Cannot prepare path leading to repository." end local ok, err = luxio.rename(self:fs_path(), newpath) if ok ~= 0 then return false, "Unable to rename repository: " .. luxio.strerror(err) end return true end function repo_method:copy_to(target) local ok, err if not target.is_nascent then return false, "Target repository is not Nascent" end local newpath = target:fs_path() -- copy to a different path so it does not appear until finished local temp_path = newpath .. ".in_progress" if not util.mkdir_p(util.dirname(temp_path)) then return false, "Cannot prepare path leading to repository." end -- attempt to create the target directory, so we can detect -- a copy is already in progress and return without removing -- the target directory ok, err = luxio.mkdir(temp_path, sio.tomode'0755') if ok ~= 0 then log.error("Failed to copy repository", self:fs_path(), "to", newpath .. ":", "Copy already in progress") return false, "Copy already in progress" end local from = self:fs_path() local function filter(parent, name, info) return parent == from and name == "objects" or util.copy_dir_filter_base(parent, name, info) end -- copy non-objects parts of the git repository ok, err = util.copy_dir(from, temp_path, nil, filter) if not ok then log.error("Failed to copy repository", from, "to", temp_path, err) util.rm_rf(temp_path) return false, "Failed to copy repository" end -- Hardlink the objects tree local cbs = util.deep_copy(util.copy_dir_copy_callbacks) cbs[luxio.DT_REG] = util.hardlink_file ok, err = util.copy_dir(util.path_join(from, 'objects'), util.path_join(temp_path, 'objects'), cbs) if not ok then log.error("Failed to hardlink objects of", from, "to", temp_path, err) util.rm_rf(temp_path) return ok, "Failed to copy repository" end -- rename into place ok, err = luxio.rename(temp_path, newpath) if ok ~= 0 then log.error("Failed to rename repository", temp_path, "to", newpath, luxio.strerror(err)) util.rm_rf(temp_path) return false, "Failed to copy repository" end return true end function repo_method:update_modified_date(shas) -- Update the info/web/last-modified local dirpath = self:fs_path() .. "/info/web" local modfile = dirpath .. "/last-modified" if not util.mkdir_p(dirpath) then return false, "Cannot prepare path leading to info file." end local last_mod_mtime = 0 local last_mod_offset = "+0000" local function update_based_on(mtime, offset) mtime = tonumber(mtime) if mtime > last_mod_mtime then last_mod_mtime = mtime last_mod_offset = offset end end local f = io.open(modfile, "r") if f then local s = f:read("*l") if s then local cur_mod_time, cur_mod_offset = s:find("^([0-9]+) ([+-][0-9]+)$") if cur_mod_time then update_based_on(cur_mod_time, cur_mod_offset) end end f:close() end for _, sha in pairs(shas) do if sha ~= string.rep("0", 40) then local obj = self.git:get(sha) if obj.type == "tag" then local tagger = obj.content.tagger update_based_on(tagger.unixtime, tagger.timezone) elseif obj.type == "commit" then local committer, author = obj.content.committer, obj.content.author update_based_on(committer.unixtime, committer.timezone) update_based_on(author.unixtime, author.timezone) end end end f = io.open(modfile, "w") if not f then return false, "Could not open info/web/last-modified for writing" end f:write(("%d %s\n"):format(last_mod_mtime, last_mod_offset)) f:close() return true end function repo_method:save_admin(reason, username) local cursha = self.git:get_ref(adminrefname) local curcommit = self.git:get(cursha) local flat_tree = gall.tree.flatten(curcommit.content.tree.content) local new_conf_content = self.project_config:serialise() local conf_blob, msg = gall.object.create(self.git, "blob", new_conf_content) if not conf_blob then return nil, msg end flat_tree["project.conf"] = conf_blob if self.readme_mdwn then local readme_blob, msg = gall.object.create(self.git, "blob", self.readme_mdwn) if not readme_blob then return nil, msg end flat_tree["README.mdwn"] = readme_blob end local tree, msg = gall.tree.create(self.git, flat_tree) if not tree then return nil, msg end local person = (username and { realname = self.config.users[username].real_name, email = self.config.users[username].email_address }) or admin_name -- Now a commit of the blob, using the gitano admin identity local commit, msg = gall.commit.create(self.git, { author = person, committer = person, message = reason or "Updated admin tree", tree = tree, parents = { curcommit } }) if not commit then return nil, msg end -- Check that it's all good local ok, msg = self:validate_admin_sha(commit.sha) if not ok then return nil, msg end -- Finally create/update the gitano/admin ref local ok, msg = self.git:update_ref(adminrefname, commit.sha, nil, curcommit.sha) if not ok then return nil, msg end return self:run_checks() end local repo_meta = { __index = repo_method, } local function check_repodir(dirname, repo) local dirp, err = luxio.opendir(dirname) if not dirp then if err == luxio.ENOENT then repo.is_nascent = true return repo else return nil, luxio.strerror(err) end else local needed = { branches = true, config = true, description = true, HEAD = true, objects = true, hooks = true, refs = true } local e, i repeat e, i = luxio.readdir(dirp) if e == 0 then needed[i.d_name] = nil end until not e if next(needed) then return nil, next(needed) .. " missing" end end return true end local function find_repository(config, reponame) -- Given the provided config, locate a repository represented by -- reponame. If the repository does not exist then the returned -- repository is in a nascent state. -- -- If the repository exists, then it is examined and brought up-to-date -- with any global config changes before being returned. log.ddebug("find_repository", reponame) -- Inject a leading '/' reponame = "/" .. reponame -- Remove any spaces, tabs, newlines or nulls reponame = reponame:gsub("[%s%z]+", "") -- Remove any '.' which follows a '/' reponame = reponame:gsub("/%.+", "/") -- simplify any sequence of '/' to a single '/' reponame = reponame:gsub("/+", "/") -- Remove any leading or trailing / reponame = reponame:match("^/*(.-)/*$") -- Remove trailing .git if present. if reponame:match(".%.git$") then reponame = reponame:match("^(.+)%.git$") end log.ddebug("find_repository", "cleaned", reponame) -- Construct the repo local repo = setmetatable({config = config, name = reponame}, repo_meta) local ok, msg = check_repodir(repo:fs_path(), repo) if not ok then return nil, msg end -- Nascent project repositories need to be returned now, no point looking -- for a git repo we can't find. if repo.is_nascent then -- Load a system lace. local lace, reason = lace.compile(repo) if not lace then return nil, reason end -- Stuff in the lace so :run_lace() work repo.lace = lace return repo end -- Let's get hold of a git repo for this local r, msg = gall.repository.new(repo:fs_path()) if not r then return nil, msg end repo.git = r -- Okay, it smells alive, so let's verify it local ok, msg = repo:run_checks() if not ok then return nil, msg end return repo end local function foreach_repository(conf, callback, filterfn) -- Scan the contents of the configured repository root finding -- potential repo names. Sort that list and then for each repo -- "find" it and call the callback with the name, repo and message local all_repos = {} local function scan_dir(dirname, prefix) local dirp, err = luxio.opendir(dirname) if not dirp then if err == luxio.ENOTDIR then return true end return nil, luxio.strerror(err) end local e, i local recurse = {} repeat e, i = luxio.readdir(dirp) if e == 0 then if i.d_name:find("%.git$") then -- Might be a repo, save for later all_repos[#all_repos+1] = (util.path_join(prefix, i.d_name) ):gsub("^/", "") else if i.d_name:find("^[^%.]") then recurse[#recurse+1] = i.d_name end end end until not e dirp = nil -- Allow GC of DIR handle -- Now try and recurse if possible, for i = 1, #recurse do local ok, msg = scan_dir(util.path_join(dirname, recurse[i]), util.path_join(prefix, recurse[i])) if not ok then return ok, msg end end return true end local ok, msg = scan_dir(config.repo_path(), "") if not ok then return ok, msg end -- Now for each repo name we think we've found, we "find" it and -- report that table.sort(all_repos) for i = 1, #all_repos do local want = true if filterfn and not filterfn(all_repos[i]) then want = false end if want then local ok, msg = find_repository(conf, all_repos[i]) callback(all_repos[i], ok, msg) end end return true end return { find = find_repository, foreach = foreach_repository, }