From 79067c7a2f194d4524e093698bb03a608c2ab8dc Mon Sep 17 00:00:00 2001 From: Shaun McCance Date: Sat, 13 Jun 2020 13:32:12 -0400 Subject: Initial work on converting yelp-check to python --- tools/yelp-check.py | 1042 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1042 insertions(+) create mode 100644 tools/yelp-check.py diff --git a/tools/yelp-check.py b/tools/yelp-check.py new file mode 100644 index 0000000..9754bd3 --- /dev/null +++ b/tools/yelp-check.py @@ -0,0 +1,1042 @@ +#!/bin/python3 +# +# yelp-check +# Copyright (C) 2011-2020 Shaun McCance +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import lxml.etree +import os +import sys +import urllib.request +import shutil +import subprocess +import tempfile + +# FIXME: don't hardcode this +DATADIR = '/usr/share/yelp-tools' + +XML_ID = '{http://www.w3.org/XML/1998/namespace}id' +NAMESPACES = { + 'mal': 'http://projectmallard.org/1.0/', + 'cache': 'http://projectmallard.org/cache/1.0/', + 'db': 'http://docbook.org/ns/docbook', + 'e': 'http://projectmallard.org/experimental/', + 'ui': 'http://projectmallard.org/ui/1.0/', + 'uix': 'http://projectmallard.org/experimental/ui/', + 'xlink': 'http://www.w3.org/1999/xlink' + } + +def _stringify(el): + ret = el.text or '' + for ch in el: + ret = ret + _stringify(ch) + if el.tail is not None: + ret = ret + el.tail + return ret + +def get_format(node): + ns = lxml.etree.QName(node).namespace + if ns in (NAMESPACES['mal'], NAMESPACES['cache']): + return 'mallard' + elif ns == NAMESPACES['db']: + return 'docbook5' + elif ns is None: + # For now, just assume no ns means docbook4 + return 'docbook4' + else: + return None + +class InputFile: + def __init__(self, filepath, filename, sitedir=None): + self.filepath = filepath + self.filename = filename + self.absfile = os.path.join(filepath, filename) + self.absdir = os.path.dirname(self.absfile) + self.sitedir = sitedir or '' + self.sitefilename = self.sitedir + self.filename + + +class Checker: + name = None + desc = None + blurb = None + formats = [] + arguments = [] + postblurb = None + xinclude = True + + def __init__(self, yelpcheck): + self.yelpcheck = yelpcheck + self.options = {} + self.fileargs = [] + self.tmpdir = None + + def __del__(self): + if self.tmpdir is not None: + shutil.rmtree(self.tmpdir) + self.tmpdir = None + + def parse_args(self, args): + while len(args) > 0: + isopt = False + for arg in self.arguments: + if arg[0] == args[0]: + if arg[1] is not None: + if len(args) < 2: + self.print_help() + return 1 + if args[0] == '--allow': + # FIXME we shouldn't just special-case --allow + self.options.setdefault(args[0], []) + self.options[args[0]].append(args[1]) + else: + self.options[args[0]] = args[1] + args = args[2:] + else: + self.options[args[0]] = True + args = args[1:] + isopt = True + break + if not isopt: + self.fileargs.append(args[0]) + args = args[1:] + return 0 + + def iter_files(self, sitedir=None): + issite = self.options.get('-s', False) + for filearg in self.fileargs: + if os.path.isdir(filearg): + if issite: + for infile in self.iter_site(filearg, '/'): + yield infile + else: + for fname in os.listdir(filearg): + if fname.endswith('.page'): + yield InputFile(filearg, fname) + else: + yield InputFile(os.getcwd(), filearg) + + def iter_site(self, filepath, sitedir): + for fname in os.listdir(filepath): + newpath = os.path.join(filepath, fname) + if os.path.isdir(newpath): + if fname == '__pintail__': + continue + for infile in self.iter_site(newpath, sitedir + fname + '/'): + yield infile + elif fname.endswith('.page'): + yield InputFile(filepath, fname, sitedir) + + def get_xml(self, xmlfile): + # FIXME: we can cache these if we add a feature to run multiple + # checkers at once + tree = lxml.etree.parse(xmlfile.absfile) + if self.xinclude: + lxml.etree.XInclude()(tree.getroot()) + return tree + + def create_tmpdir(self): + if self.tmpdir is None: + self.tmpdir = tempfile.mkdtemp() + + def print_help(self): + print('Usage: yelp-check ' + self.name + ' [OPTIONS] [FILES]') + print('Formats: ' + ' '.join(self.formats) + '\n') + #FIXME: prettify names of formats + print(self.blurb) + print('\nOptions:') + maxarglen = 2 + for arg in self.arguments: + if arg[1] is not None: + maxarglen = max(len(arg[1]) + len(arg[0]) + 1, maxarglen) + for arg in self.arguments: + arg1 = (' ' + arg[1]) if arg[1] is not None else '' + print(' ' + (arg[0] + arg1).ljust(maxarglen) + ' ' + arg[2]) + if self.postblurb is not None: + print(self.postblurb) + + def main(self, args): + pass + + +class HrefsChecker (Checker): + name = 'hrefs' + desc = 'Find broken external links in a document' + blurb = ('Find broken href links in FILES in a Mallard document, or\n' + + 'broken ulink or XLink links in FILES in a DocBook document.') + formats = ['docbook4', 'docbook5', 'mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site')] + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + # safelisting URLs that we use as identifiers + hrefs = { + 'http://creativecommons.org/licenses/by-sa/3.0/': True, + 'http://creativecommons.org/licenses/by-sa/3.0/us/': True + } + retcode = 0 + + for infile in self.iter_files(): + xml = self.get_xml(infile) + for el in xml.xpath('//*[@href | @xlink:href | self::ulink/@url]', + namespaces=NAMESPACES): + href = el.get('href', None) + if href is None: + href = el.get('{www.w3.org/1999/xlink}href') + if href is None: + href = el.get('url') + if href is None: + continue + if href.startswith('mailto:'): + continue + if href not in hrefs: + try: + req = urllib.request.urlopen(href) + hrefs[href] = (req.status == 200) + except Exception as e: + hrefs[href] = False + if not hrefs[href]: + retcode = 1 + print(infile.sitefilename + ': ' + href) + + return retcode + + +class IdsChecker (Checker): + name = 'ids' + desc = 'Find Mallard page IDs that do not match file names' + blurb = ('Find pages in a Mallard document whose page ID does not match\n' + + 'the base file name of the page file.') + formats = ['mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site')] + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + retcode = 0 + + for infile in self.iter_files(): + xml = self.get_xml(infile) + isok = False + pageid = None + if infile.filename.endswith('.page'): + try: + pageid = xml.getroot().get('id') + isok = (pageid == os.path.basename(infile.filename)[:-5]) + except: + isok = False + if not isok: + retcode = 1 + print(infile.sitefilename + ': ' + (pageid or '')) + + return retcode + + +class LinksChecker (Checker): + name = 'links' + desc = 'Find broken xref or linkend links in a document' + blurb = ('Find broken xref links in FILES in a Mallard document,\n' + + 'or broken linkend links in FILES in a DocBook document.') + formats = ['docbook4', 'docbook5', 'mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site'), + ('-c', 'CACHE', 'Use the existing Mallard cache CACHE'), + ('-i', None, 'Ignore xrefs where href is present')] + + def __init__(self, yelpcheck): + super().__init__(yelpcheck) + self.idstoxrefs = {} + self.idstolinkends = {} + + def _accumulate_mal(self, node, pageid, sectid, xrefs, sitedir=None): + thisid = node.get('id') + if thisid is not None: + if node.tag == '{' + NAMESPACES['mal'] + '}page': + pageid = thisid + else: + sectid = thisid + curid = pageid + if curid is not None: + if sectid is not None: + # id attrs in cache files are already fully formed + if '#' in sectid: + curid = sectid + else: + curid = curid + '#' + sectid + if sitedir is not None: + # id attrs in cache files already have sitedir prefixed + if curid[0] != '/': + curid = sitedir + curid + self.idstoxrefs.setdefault(curid, []) + if xrefs: + xref = node.get('xref') + if xref is not None: + self.idstoxrefs[curid].append(xref) + for child in node: + self._accumulate_mal(child, pageid, sectid, xrefs, sitedir) + + def _accumulate_db(self, node, nodeid): + thisid = node.get('id') + if thisid is None: + thisid = node.get(XML_ID) + if thisid is not None: + nodeid = thisid + self.idstolinkends.setdefault(nodeid, []) + if nodeid is not None: + linkend = node.get('linkend') + if linkend is not None: + self.idstolinkends[nodeid].append(linkend) + for child in node: + self._accumulate_db(child, nodeid) + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + retcode = 0 + + if '-c' in self.options: + xml = self.get_xml(InputFile(os.getcwd(), self.options['-c'])) + self._accumulate_mal(xml.getroot(), None, None, False) + + for infile in self.iter_files(): + xml = self.get_xml(infile) + format = get_format(xml.getroot()) + if format == 'mallard': + self._accumulate_mal(xml.getroot(), None, None, True, infile.sitedir) + elif format in ('docbook4', 'docbook5'): + # For DocBook, we assume each filearg is its own document, so + # we reset the dict each time and only check within the file. + # Note that XInclude and SYSTEM includes DO happen first. + self.idstolinkends = {} + self._accumulate_db(xml.getroot(), None) + for curid in self.idstolinkends: + for linkend in self.idstolinkends[curid]: + if linkend not in self.idstolinkends: + print(curid + ': ' + linkend) + retcode = 1 + + for curid in self.idstoxrefs: + for xref in self.idstoxrefs[curid]: + checkref = xref + if checkref[0] == '#': + checkref = curid.split('#')[0] + checkref + if curid[0] == '/' and checkref[0] != '/': + checkref = curid[:curid.rfind('/')+1] + checkref + if checkref not in self.idstoxrefs: + print(curid + ': ' + xref) + retcode = 1 + + return retcode + + +class MediaChecker (Checker): + name = 'media' + desc = 'Find broken references to media files' + blurb = ('Find broken references to media files. In Mallard, this\n' + + 'checks media and thumb elements. In DocBook, this checks\n' + + 'audiodata, imagedata, and videodata elements.') + formats = ['docbook4', 'docbook5', 'mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site')] + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + retcode = 0 + + for infile in self.iter_files(): + xml = self.get_xml(infile) + format = get_format(xml.getroot()) + srcs = [] + if format == 'mallard': + for el in xml.xpath('//mal:media[@src] | //uix:thumb | //ui:thumb | //e:mouseover', + namespaces=NAMESPACES): + srcs.append(el.get('src')) + elif format == 'docbook5': + # FIXME: do we care about entityref? + for el in xml.xpath('//db:audiodata | //db:imagedata | //db:videodata', + namespaces=NAMESPACES): + srcs.append(el.get('fileref')) + elif format == 'docbook4': + for el in xml.xpath('//audiodata | //imagedata | //videodata'): + srcs.append(el.get('fileref')) + for src in srcs: + fsrc = os.path.join(infile.absdir, src) + if not os.path.exists(fsrc): + print(infile.sitefilename + ': ' + src) + retcode = 1 + + return retcode + + +class OrphansChecker (Checker): + name = 'orphans' + desc = 'Find orphaned pages in a Mallard document' + blurb = ('Locate orphaned pages among FILES in a Mallard document.\n' + + 'Orphaned pages are any pages that cannot be reached by\n' + + 'topic links alone from the index page.') + formats = ['mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site'), + ('-c', 'CACHE', 'Use the existing Mallard cache CACHE')] + + def __init__(self, yelpcheck): + super().__init__(yelpcheck) + self.guidelinks = {} + self.sitesubdirs = set() + + def _collect_links(self, node, sitedir): + pageid = node.get('id') + if pageid[0] != '/': + # id attrs in cache files already have sitedir prefixed + pageid = sitedir + pageid + else: + sitedir = pageid[:pageid.rfind('/')+1] + self.guidelinks.setdefault(pageid, set()) + # For the purposes of finding orphans, we'll just pretend that + # all links to or from sections are just to or from pages. + for el in node.xpath('//mal:info/mal:link[@type="guide"]', + namespaces=NAMESPACES): + xref = el.get('xref') + if xref is None or xref == '': + continue + if xref[0] == '#': + continue + if '#' in xref: + xref = xref[:xref.find('#')] + if sitedir is not None and sitedir != '': + if xref[0] != '/': + xref = sitedir + xref + self.guidelinks[pageid].add(xref) + for el in node.xpath('//mal:info/mal:link[@type="topic"]', + namespaces=NAMESPACES): + xref = el.get('xref') + if xref is None or xref == '': + continue + if xref[0] == '#': + continue + if '#' in xref: + xref = xref[:xref.find('#')] + if sitedir is not None and sitedir != '': + if xref[0] != '/': + xref = sitedir + xref + self.guidelinks.setdefault(xref, set()) + self.guidelinks[xref].add(pageid) + for el in node.xpath('//mal:links[@type="site-subdirs" or @type="site:subdirs"]', + namespaces=NAMESPACES): + self.sitesubdirs.add(pageid) + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + retcode = 0 + + if '-c' in self.options: + xml = self.get_xml(InputFile(os.getcwd(), self.options['-c'])) + for page in xml.getroot(): + if page.tag == '{' + NAMESPACES['mal'] + '}page': + pageid = page.get('id') + if pageid is None or pageid == '': + continue + self._collect_links(page) + + pageids = set() + for infile in self.iter_files(): + xml = self.get_xml(infile) + pageid = xml.getroot().get('id') + if pageid is None: + continue + pageids.add(infile.sitedir + pageid) + self._collect_links(xml.getroot(), infile.sitedir) + + siteupdirs = {} + for pageid in self.sitesubdirs: + dirname = pageid[:pageid.rfind('/')+1] + for subid in self.guidelinks: + if subid.startswith(dirname): + if subid.endswith('/index'): + mid = subid[len(dirname):-6] + if mid != '' and '/' not in mid: + siteupdirs[subid] = pageid + + if '-s' in self.options: + okpages = set(['/index']) + else: + okpages = set(['index']) + for pageid in sorted(pageids): + if pageid in okpages: + isok = True + else: + isok = False + guides = [g for g in self.guidelinks[pageid]] + if pageid in siteupdirs: + updir = siteupdirs[pageid] + if updir not in guides: + guides.append(updir) + cur = 0 + while cur < len(guides): + if guides[cur] in okpages: + isok = True + break + if guides[cur] in self.guidelinks: + for guide in self.guidelinks[guides[cur]]: + if guide not in guides: + guides.append(guide) + cur += 1 + if isok: + okpages.add(pageid) + else: + print(pageid) + retcode = 1 + + return retcode + + +class ValidateChecker (Checker): + name = 'validate' + desc = 'Validate files against a DTD or RNG' + blurb = ('Validate FILES against the appropriate DTD or RNG.\n' + + 'For Mallard pages, perform automatic RNG merging\n' + + 'based on the version attribute.') + formats = ['docbook4', 'docbook5', 'mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site'), + ('--strict', None, 'Disallow unknown namespaces'), + ('--allow', 'NS', 'Explicitly allow namespace NS in strict mode'), + ('--jing', None, 'Use jing instead of xmllint for RNG validation')] + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + retcode = 0 + + for infile in self.iter_files(): + xml = self.get_xml(infile) + format = get_format(xml.getroot()) + command = None + if format == 'mallard': + version = xml.getroot().get('version') + if version is None or version == '': + tag = xml.getroot().tag + if tag == '{' + NAMESPACES['mal'] + '}stack': + # 1.2 isn't final yet as of 2020-01-09. Stacks will + # likely be in 1.2, so we can assume at least that. + version = '1.2' + elif tag == '{' + NAMESPACES['cache'] + '}cache': + version = 'cache/1.0' + else: + version = '1.0' + self.create_tmpdir() + rng = os.path.join(self.tmpdir, + version.replace('/', '__').replace(' ', '__')) + if not os.path.exists(rng): + strict = 'true()' if ('--strict' in self.options) else 'false()' + if '--allow' in self.options: + allow = ' '.join(self.options['--allow']) + else: + allow = ' ' + subprocess.call(['xsltproc', '-o', rng, + '--param', 'rng.strict', strict, + '--stringparam', 'rng.strict.allow', allow, + os.path.join(DATADIR, 'xslt', 'mal-rng.xsl'), + infile.absfile]) + if '--jing' in self.options: + command = ['jing', '-i', rng, infile.filename] + else: + command = ['xmllint', '--noout', '--xinclude', '--noent', + '--relaxng', rng, infile.filename] + elif format == 'docbook4': + if xml.docinfo.doctype.startswith('' + print('From: ' + name) + date = ch.get('date') + if date is not None: + print('Date: ' + date) + print('') + for ch in el: + if isinstance(ch, lxml.etree._ProcessingInstruction): + continue + elif ch.tag == '{' + NAMESPACES['mal'] + '}cite': + continue + elif ch.tag in ('{' + NAMESPACES['mal'] + '}p', + '{' + NAMESPACES['mal'] + '}title'): + for s in _stringify(ch).strip().split('\n'): + print(' ' + s.strip()) + print('') + else: + name = lxml.etree.QName(ch).localname + print(' <' + name + '>...\n') + elif format in ('docbook4', 'docbook5'): + if format == 'docbook4': + dbxpath = '//remark' + else: + dbxpath = '//db:remark' + for el in xml.xpath(dbxpath, namespaces=NAMESPACES): + thisid = infile.filename + par = el + while par is not None: + sectid = par.get('id') + if sectid is None: + sectid = par.get(XML_ID) + if sectid is not None: + thisid = thisid + '#' + sectid + break + par = par.getparent() + print('Page: ' + thisid) + flag = el.get('revisionflag') + if flag is not None: + print('Flag: ' + flag) + print('') + for s in _stringify(el).strip().split('\n'): + print(' ' + s.strip()) + print('') + + return 0 + + +class LicenseChecker (Checker): + name = 'license' + desc = 'Report the license of Mallard pages' + blurb = ('Report the license of the Mallard page files FILES. Each\n' + + 'matching page is reporting along with its license, reported\n' + + 'based on the href attribute of the license element. Common\n' + + 'licenses use a shortened identifier. Pages with multiple\n' + + 'licenses have the identifiers separated by spaces. Pages\n' + + 'with no license element report \'none\'. Licenses with no\n' + + 'href attribute are reported as \'unknown\'') + formats = ['mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site'), + ('--only', 'LICENSES', 'Only show pages whose license is in LICENSES'), + ('--except', 'LICENSES', 'Exclude pages whose license is in LICENSES'), + ('--totals', None, 'Show total counts for each license')] + postblurb = 'LICENSES may be a comma- and/or space-separated list.' + + def get_license(self, href): + if href is None: + return 'unknown' + elif (href.startswith('http://creativecommons.org/licenses/') or + href.startswith('https://creativecommons.org/licenses/')): + return 'cc-' + '-'.join([x for x in href.split('/') if x][3:]) + elif (href.startswith('http://www.gnu.org/licenses/') or + href.startswith('https://www.gnu.org/licenses/')): + return href.split('/')[-1].replace('.html', '') + else: + return 'unknown' + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + totals = {} + + for infile in self.iter_files(): + xml = self.get_xml(infile) + thisid = xml.getroot().get('id') or infile.filename + licenses = [] + for el in xml.xpath('/mal:page/mal:info/mal:license', + namespaces=NAMESPACES): + licenses.append(self.get_license(el.get('href'))) + if len(licenses) == 0: + licenses.append('none') + + if '--only' in self.options: + only = self.options['--only'].replace(',', ' ').split() + skip = True + for lic in licenses: + if lic in only: + skip = False + if skip: + continue + if '--except' in self.options: + cept = self.options['--except'].replace(',', ' ').split() + skip = False + for lic in licenses: + if lic in cept: + skip = True + if skip: + continue + + if '--totals' in self.options: + for lic in licenses: + totals.setdefault(lic, 0) + totals[lic] += 1 + else: + print(infile.sitedir + thisid + ': ' + ' '.join(licenses)) + + if '--totals' in self.options: + for lic in sorted(totals): + print(lic + ': ' + str(totals[lic])) + + return 0 + + +class StatusChecker (Checker): + name = 'status' + desc = 'Report the status of Mallard pages' + blurb = ('Report the status of the Mallard page files FILES. Each\n' + + 'matching page is reporting along with its status.') + formats = ['mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site'), + ('--version', 'VER', 'Select revisions with the version attribute VER'), + ('--docversion', 'VER', 'Select revisions with the docversion attribute VER'), + ('--pkgversion', 'VER', 'Select revisions with the pkgversion attribute VER'), + ('--older', 'DATE', 'Only show pages older than DATE'), + ('--newer', 'DATE', 'Only show pages newer than DATE'), + ('--only', 'STATUSES', 'Only show pages whose status is in STATUSES'), + ('--except', 'STATUSES', 'Exclude pages whose status is in STATUSES'), + ('--totals', None, 'Show total counts for each status')] + postblurb = 'VER and STATUSES may be comma- and/or space-separated lists.' + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + totals = {} + + for infile in self.iter_files(): + xml = self.get_xml(infile) + pageid = xml.getroot().get('id') + bestrev = None + for rev in xml.xpath('/mal:page/mal:info/mal:revision', namespaces=NAMESPACES): + revversion = (rev.get('version') or '').split() + docversion = rev.get('docversion') + if docversion is not None: + revversion.append('doc:' + docversion) + pkgversion = rev.get('pkgversion') + if pkgversion is not None: + revversion.append('pkg:' + pkgversion) + checks = [] + if '--version' in self.options: + checks.append(self.options['--version'].replace(',', ' ').split()) + if '--docversion' in self.options: + checks.append(['doc:' + s for s in self.options['--docversion'].replace(',', ' ').split()]) + if '--pkgversion' in self.options: + checks.append(['pkg:' + s for s in self.options['--pkgversion'].replace(',', ' ').split()]) + revok = True + for check in checks: + checkok = False + for v in check: + if v in revversion: + checkok = True + break + if not checkok: + revok = False + break + if revok: + if bestrev is None: + bestrev = rev + continue + bestdate = bestrev.get('date') + thisdate = rev.get('date') + if bestdate is None: + bestrev = rev + elif thisdate is None: + pass + elif thisdate >= bestdate: + bestrev = rev + if bestrev is not None: + status = bestrev.get('status') or 'none' + date = bestrev.get('date') or None + else: + status = 'none' + date = None + if '--older' in self.options: + if date is None or date >= self.options['--older']: + continue + if '--newer' in self.options: + if date is None or date <= self.options['--newer']: + continue + if '--only' in self.options: + if not status in self.options['--only'].replace(',', ' ').split(): + continue + if '--except' in self.options: + if status in self.options['--except'].replace(',', ' ').split(): + continue + if '--totals' not in self.options: + print(infile.sitedir + pageid + ': ' + status) + else: + totals.setdefault(status, 0) + totals[status] += 1 + + if '--totals' in self.options: + for st in sorted(totals): + print(st + ': ' + str(totals[st])) + + return 0 + + +class StyleChecker (Checker): + name = 'style' + desc = 'Report the style attribute of Mallard pages' + blurb = ('Report the page style attribute of the Mallard page files\n' + + 'FILES. Each matching page is reporting along with its status.') + formats = ['mallard'] + arguments = [ + ('-h', None, 'Show this help and exit'), + ('-s', None, 'Treat pages as belonging to a Mallard site'), + ('--only', 'STYLES', 'Only show pages whose style is in STATUSES'), + ('--except', 'STYLES', 'Exclude pages whose style is in STATUSES'), + ('--totals', None, 'Show total counts for each style')] + postblurb = 'STYLES may be comma- and/or space-separated lists.' + + def main(self, args): + if self.parse_args(args) != 0: + return 1 + if '-h' in self.options: + self.print_help() + return 0 + + totals = {} + + for infile in self.iter_files(): + xml = self.get_xml(infile) + thisid = xml.getroot().get('id') + style = xml.getroot().get('style') + if style is None: + style = 'none' + styles = style.split() + # We'll set style to None if it doesn't meat the criteria + if '--only' in self.options: + only = self.options['--only'].replace(',', ' ').split() + if len(only) == 0: + # We treat a blank --only as requesting pages with no style + if style != 'none': + style = None + else: + allow = False + for st in styles: + if st in only: + allow = True + break + if not allow: + style = None + if '--except' in self.options: + cept = self.options['--except'].replace(',', ' ').split() + for st in styles: + if st in cept: + style = None + break + if '--totals' not in self.options: + if style is not None: + print(infile.sitedir + thisid + ': ' + style) + else: + if style is not None: + for st in styles: + totals.setdefault(st, 0) + totals[st] += 1 + + if '--totals' in self.options: + for st in sorted(totals): + print(st + ': ' + str(totals[st])) + + return 0 + + +class YelpCheck: + def __init__(self): + pass + + def main(self): + if len(sys.argv) < 2: + self.print_usage() + return 1 + + checker = None + for cls in Checker.__subclasses__(): + if sys.argv[1] == cls.name: + checker = cls + + if checker is None: + self.print_usage() + return 1 + + return checker(self).main(sys.argv[2:]) + + def print_usage(self): + print('Usage: yelp-check [OPTIONS] [FILES]') + namelen = 2 + checks = [] + reports = [] + others = [] + for cls in sorted(Checker.__subclasses__(), key=(lambda cls: cls.name)): + namelen = max(namelen, len(cls.name) + 2) + if cls in (HrefsChecker, IdsChecker, LinksChecker, + MediaChecker, OrphansChecker, ValidateChecker): + checks.append(cls) + elif cls in (CommentsChecker, LicenseChecker, StatusChecker, + StyleChecker): + reports.append(cls) + else: + others.append(cls) + if len(checks) > 0: + print('\nCheck commands:') + for cls in checks: + print(' ' + cls.name.ljust(namelen) + cls.desc) + if len(reports) > 0: + print('\nReport commands:') + for cls in reports: + print(' ' + cls.name.ljust(namelen) + cls.desc) + if len(others) > 0: + print('\nOther commands:') + for cls in others: + print(' ' + cls.name.ljust(namelen) + cls.desc) + + +if __name__ == '__main__': + try: + sys.exit(YelpCheck().main()) + except KeyboardInterrupt: + sys.exit(1) + -- cgit v1.2.1