libnm, libnm-util: move settings doc generation to libnm-core

Move the settings/plugins doc generation from libnm-util to
libnm-core, since libnm-util isn't being updated for all new
properties.

With this commit, the keyfile and ifcfg-rh documentation is basically
unchanged, except that deprecated properties are now gone, and new
properties have been added, and the sections are in a different order.
(generate-plugin-docs.pl just outputs the settings in Makefile order,
and they were unsorted in libnm-util, but are sorted in libnm-core).

The settings documentation used for nm-settings.5, the D-Bus API docs,
and the nmcli help is changed a bit more at this point, and mostly for
the worse, since the libnm-core setting properties don't match up with
the D-Bus API as well as the libnm-util ones do. To be fixed...

(I also removed the "plugins docs" line in each plugin docs comment
block while moving them, since those blocks will be used for more than
just plugins soon, and it's sort of obvious anyway.)

3 files changed, 433 insertions, 0 deletions

diff --git a/libnm/Makefile.am b/libnm/Makefile.am
index f911d86e66..e00aa87392 100644
--- a/libnm/Makefile.am
+++ b/libnm/Makefile.am
@@ -181,3 +181,28 @@ endif
 
 check-local:
 	$(top_srcdir)/tools/check-exports.sh $(builddir)/.libs/libnm.so $(SYMBOL_VIS_FILE)
+
+if BUILD_SETTING_DOCS
+
+noinst_DATA = nm-setting-docs.xml nm-keyfile-docs.xml nm-ifcfg-rh-docs.xml
+
+docs_sources = $(filter-out nm-core-enum-types.c,$(libnm_core_sources))
+
+nm-setting-docs.xml: generate-setting-docs.py $(docs_sources) | NM-1.0.gir NM-1.0.typelib libnm.la
+	export GI_TYPELIB_PATH=$(abs_builddir)$${GI_TYPELIB_PATH:+:$$GI_TYPELIB_PATH}; \
+	export LD_LIBRARY_PATH=$(abs_builddir)/.libs$${LD_LIBRARY_PATH:+:$$LD_LIBRARY_PATH}; \
+	$(srcdir)/generate-setting-docs.py \
+		--gir $(builddir)/NM-1.0.gir \
+		--output $@
+
+nm-keyfile-docs.xml: generate-plugin-docs.pl $(docs_sources)
+	$(srcdir)/generate-plugin-docs.pl keyfile $(top_srcdir)/libnm-core $@
+nm-ifcfg-rh-docs.xml: generate-plugin-docs.pl $(docs_sources)
+	$(srcdir)/generate-plugin-docs.pl ifcfg-rh $(top_srcdir)/libnm-core $@
+
+CLEANFILES += $(noinst_DATA)
+EXTRA_DIST += $(noinst_DATA)
+
+endif
+
+EXTRA_DIST += generate-setting-docs.py generate-plugin-docs.pl
diff --git a/libnm/generate-plugin-docs.pl b/libnm/generate-plugin-docs.pl
new file mode 100755
index 0000000000..53edb48cd2
--- /dev/null
+++ b/libnm/generate-plugin-docs.pl
@@ -0,0 +1,198 @@
+#!/usr/bin/env perl
+# vim: ft=perl ts=2 sts=2 sw=2 et ai
+# -*- Mode: perl; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#
+# Copyright 2014 Red Hat, Inc.
+#
+
+#
+# The script parses nm-setting-*.c files and extracts documentation related
+# to setting plugins. The documentation is in a simple format of lines
+# "keyword: value". The documentation is enclosed between tags
+# ---<plugin-name>--- and ---end---
+# Recognized keywords are:
+# "property: "     - property name
+# "variable: "     - name of the variable used by the plugin
+# "format: "       - format of the value in 'keyfile' plugin
+# "default: "      - default value when variable is not used
+# "values: "       - allowed values (e.g. for enumerations)
+# "example: "      - example(s)
+# "description: "  - description text
+# Value is an arbitrary string that can span over multiple lines.
+#
+# ifcfg-rh specifics:
+#  - mark NM extension variables with (+), e.g. variable: UUID(+)
+#
+
+use strict;
+use warnings;
+use v5.10;
+#YAML:XS is based on libyaml C library and it is a good and fast YAML implementation.
+#However it may not be present everywhere. So use YAML instead.
+#use YAML::XS qw(Load);
+use YAML qw(Load);
+
+# global variables
+my @keywords = ("property", "variable", "format", "values", "default", "example", "description");
+my @source_files;
+my @data;
+my $fo;
+
+(scalar @ARGV == 3) or die "Usage: $0 <plugin> <srcdir> <output-xml-file>\n";
+($ARGV[0] eq "keyfile" || $ARGV[0] eq "ifcfg-rh") or die "Allowed <plugin> values: keyfile, ifcfg-rh\n";
+my ($plugin, $srcdir, $output) = @ARGV;
+my $start_tag = "---$plugin---\\s*\$";
+my $end_tag   = '---end---';
+
+# get source files to scan for documentation comments (nm-setting-<something>.c)
+my $file = "$srcdir/Makefile.libnm-core";
+open my $fh, '<', $file or die "Can't open $file: $!";
+while (my $line = <$fh>) {
+  chomp $line;
+  my @strings = $line =~ /\/(nm-setting-[^.]*\.c)(?:\s|$)/g;
+  push @source_files, @strings
+}
+close $fh;
+
+# open output file
+open $fo, '>', $output or die "Can't open $output: $!";
+
+# write XML header
+write_header();
+
+# write generated documenation for each setting
+foreach my $c_file (@source_files) {
+  my $path = "$srcdir/$c_file";
+  my $setting_name = get_setting_name($path);
+  write_item("<setting name=\"$setting_name\">");
+  scan_doc_comments($path, $start_tag, $end_tag);
+  write_item("</setting>");
+}
+
+# write XML footer
+write_footer();
+
+# close output file
+close $fo;
+
+
+### --- subroutines --- ###
+
+# get setting name from NM_SETTING_*_SETTING_NAME constant in C header file
+sub get_setting_name {
+  my $path = $_[0];
+  $path =~ s/c$/h/;  # use header file to find out setting name
+  open my $fh, '<', $path or die "Can't open $path: $!";
+  while (my $line = <$fh>) {
+    if ($line =~ /NM_SETTING_.+SETTING_NAME\s+\"(\S+)\"/) {
+      return $1;
+    }
+  }
+}
+
+# scan source setting file for documentation tags and write them to XML
+sub scan_doc_comments {
+  my($setting_file, $start, $end) = @_;
+  open my $fi, '<', $setting_file or die "Can't open $setting_file: $!";
+  while (<$fi>) {
+    if (/$start/ .. /$end/) {
+      next if /$start/;
+      if (/$end/) {
+        process_data();
+      } else {
+        push @data, $_;
+      }
+      next;
+    }
+    # ignore text not inside marks
+  }
+  close $fi;
+}
+
+# process plugin property documentation comments (as a YAML document)
+sub process_data {
+  return if not @data;
+  my $kwd_pat = join("|", @keywords);
+  my $yaml_literal_seq = "|\n";
+
+  foreach (@data) {
+    # make a proper YAML document from @data
+    $_ =~ s/^\s*\**\s+|\s+$//;  # remove leading spaces and *, and traling spaces
+    # Properly indent the text so that it is a valid YAML, and insert | (for literal text)
+    if ($_ =~ /^($kwd_pat):\s+/) {
+      # add | after "keyword:" that allows using literal text (YAML won't break on special character)
+      # http://learnxinyminutes.com/docs/yaml/ and http://www.yaml.org/spec/1.2/spec.html#id2795688
+      $_ =~ s/(^($kwd_pat):)/$1 $yaml_literal_seq/;
+    } else {
+      $_ = " " . $_;  # indent the text
+    }
+  }
+  my $str = join ("", @data);
+  my $yaml_data = Load($str);
+
+  # now write ia line into the XML
+  my $name   = $yaml_data->{property}    // "";
+  my $var    = $yaml_data->{variable}    // $name;  # fallback to "property: "
+  my $format = $yaml_data->{format}      // "";
+  my $values = $yaml_data->{values}      // "";
+  my $def    = $yaml_data->{default}     // "";
+  my $exam   = $yaml_data->{example}     // "";
+  my $desc   = $yaml_data->{description} // "";
+
+  escape_xml_chars($name, $var, $format, $values, $def, $exam, $desc);
+  my $foo = sprintf("<property name=\"%s\" variable=\"%s\" format=\"%s\" values=\"%s\" ".
+                    "default=\"%s\" example=\"%s\" description=\"%s\"/>",
+                    $name, $var, $format, $values, $def, $exam, $desc);
+  write_item($foo);
+  @data = ();
+}
+
+# - XML handling -
+sub write_header {
+  (my $header =
+    qq{<?xml version=\"1.0\"?>
+       <!DOCTYPE nm-$plugin-docs [
+       ]>
+
+       <nm-$plugin-docs>
+  }) =~ s/^ {7}//mg;
+  print {$fo} $header;
+}
+
+sub write_footer {
+  my $footer = "</nm-$plugin-docs>";
+  print {$fo} $footer;
+}
+
+sub write_item {
+  my $str = join("", @_);
+  print {$fo} $str, "\n";
+}
+
+sub escape_xml_chars {
+  # http://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references#Predefined%5Fentities%5Fin%5FXML
+  foreach my $val (@_) {
+    $val =~ s/&/&amp;/sg;
+    $val =~ s/</&lt;/sg;
+    $val =~ s/>/&gt;/sg;
+    $val =~ s/"/&quot;/sg;
+    $val =~ s/'/&apos;/sg;
+  }
+}
+
diff --git a/libnm/generate-setting-docs.py b/libnm/generate-setting-docs.py
new file mode 100755
index 0000000000..ef2918ec55
--- /dev/null
+++ b/libnm/generate-setting-docs.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301 USA.
+#
+# Copyright 2009 - 2014 Red Hat, Inc.
+
+from __future__ import print_function
+
+from gi.repository import NM, GObject
+import argparse, datetime, re, sys
+import xml.etree.ElementTree as ET
+
+type_name_map = {
+    'gchararray': 'string',
+    'GSList_gchararray_': 'array of string',
+    'GArray_guchar_': 'byte array',
+    'gboolean': 'boolean',
+    'guint64': 'uint64',
+    'gint': 'int32',
+    'guint': 'uint32',
+    'GArray_guint_': 'array of uint32',
+    'GPtrArray_GArray_guint__': 'array of array of uint32',
+    'GPtrArray_GArray_guchar__': 'array of byte array',
+    'GPtrArray_gchararray_': 'array of string',
+    'GHashTable_gchararray+gchararray_': 'dict of (string::string)',
+    'GPtrArray_GValueArray_GArray_guchar_+guint+GArray_guchar___': 'array of (byte array, uint32, byte array)',
+    'GPtrArray_GValueArray_GArray_guchar_+guint+GArray_guchar_+guint__': 'array of (byte array, uint32, byte array, uint32)'
+}
+
+ns_map = {
+    'c':    'http://www.gtk.org/introspection/c/1.0',
+    'gi':   'http://www.gtk.org/introspection/core/1.0',
+    'glib': 'http://www.gtk.org/introspection/glib/1.0'
+}
+identifier_key = '{%s}identifier' % ns_map['c']
+nick_key = '{%s}nick' % ns_map['glib']
+symbol_prefix_key = '{%s}symbol-prefix' % ns_map['c']
+
+constants = {
+    'TRUE': 'TRUE',
+    'FALSE': 'FALSE',
+    'G_MAXUINT32': 'G_MAXUINT32',
+    'NULL': 'NULL' }
+setting_names = {}
+
+def init_constants(girxml, settings):
+    for const in girxml.findall('./gi:namespace/gi:constant', ns_map):
+        cname = const.attrib['{%s}type' % ns_map['c']]
+        cvalue = const.attrib['value']
+        if const.find('./gi:type[@name="utf8"]', ns_map) is not None:
+            cvalue = '"%s"' % cvalue
+        constants[cname] = cvalue
+
+    for enum in girxml.findall('./gi:namespace/gi:enumeration', ns_map):
+        for enumval in enum.findall('./gi:member', ns_map):
+            cname = enumval.attrib[identifier_key]
+            cvalue = '%s (%s)' % (cname, enumval.attrib['value'])
+            constants[cname] = cvalue
+
+    for enum in girxml.findall('./gi:namespace/gi:bitfield', ns_map):
+        for enumval in enum.findall('./gi:member', ns_map):
+            cname = enumval.attrib[identifier_key]
+            cvalue = '%s (0x%x)' % (cname, int(enumval.attrib['value']))
+            constants[cname] = cvalue
+
+    for setting in settings:
+        setting_type_name = 'NM' + setting.attrib['name'];
+        setting_name_symbol = 'NM_' + setting.attrib[symbol_prefix_key].upper() + '_SETTING_NAME'
+        if constants.has_key(setting_name_symbol):
+            setting_name = constants[setting_name_symbol]
+            setting_names[setting_type_name] = setting_name
+
+def get_prop_type(setting, pspec, propxml):
+    prop_type = pspec.value_type.name
+    if prop_type in type_name_map:
+        prop_type = type_name_map[prop_type]
+    if prop_type is None:
+        prop_type = ''
+    return prop_type
+
+def get_docs(setting, pspec, propxml):
+    doc_xml = propxml.find('gi:doc', ns_map)
+    if doc_xml is None:
+        return None
+
+    doc = doc_xml.text
+    if 'deprecated' in propxml.attrib:
+        doc = doc + ' Deprecated: ' + propxml.attrib['deprecated']
+
+    doc = re.sub(r'\n\s*', r' ', doc)
+
+    # Expand constants
+    doc = re.sub(r'%([^%]\w*)', lambda match: constants[match.group(1)], doc)
+
+    # #NMSettingWired:mac-address -> "mac-address"
+    doc = re.sub(r'#[A-Za-z0-9_]*:([A-Za-z0-9_-]*)', r'"\1"', doc)
+
+    # #NMSettingWired setting -> "802-3-ethernet" setting
+    doc = re.sub(r'#([A-Z]\w*) setting', lambda match: setting_names[match.group(1)] + ' setting', doc)
+
+    # remaining gtk-doc cleanup
+    doc = doc.replace('%%', '%')
+    doc = doc.replace('<!-- -->', '')
+    doc = re.sub(r' Element-.ype:.*', '', doc)
+    doc = re.sub(r'#([A-Z]\w*)', r'\1', doc)
+
+    # Remove sentences that refer to functions
+    doc = re.sub(r'\.\s+[^.]*\w\(\)[^.]*\.', r'.', doc)
+
+    return doc
+
+def get_default_value(setting, pspec, propxml):
+    default_value = setting.get_property(pspec.name.replace('-', '_'))
+    if default_value is None:
+        return default_value
+
+    value_type = get_prop_type(setting, pspec, propxml)
+    if value_type == 'string' and default_value != '' and pspec.name != 'name':
+        default_value = '"%s"' % default_value
+    elif value_type == 'gchar' and default_value != '':
+        default_value = "'%s'" % default_value
+    elif value_type == 'boolean':
+        default_value = str(default_value).upper()
+    elif value_type == 'byte array':
+        default_value = '[]'
+    elif str(default_value).startswith('<'):
+        default_value = None
+
+    return default_value
+
+def escape(val):
+    return str(val).replace('"', '&quot;')
+
+def usage():
+    print("Usage: %s --gir FILE --output FILE" % sys.argv[0])
+    exit()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-g', '--gir', metavar='FILE', help='NM-1.0.gir file')
+parser.add_argument('-o', '--output', metavar='FILE', help='output file')
+
+args = parser.parse_args()
+if args.gir is None or args.output is None:
+    usage()
+
+NM.utils_init()
+
+girxml = ET.parse(args.gir).getroot()
+outfile = open(args.output, mode='w')
+
+basexml = girxml.find('./gi:namespace/gi:class[@name="Setting"]', ns_map)
+settings = girxml.findall('./gi:namespace/gi:class[@parent="Setting"]', ns_map)
+# Hack. Need a better way to do this
+ipxml = girxml.find('./gi:namespace/gi:class[@name="SettingIPConfig"]', ns_map)
+settings.extend(girxml.findall('./gi:namespace/gi:class[@parent="SettingIPConfig"]', ns_map))
+settings = sorted(settings, key=lambda setting: setting.attrib['{%s}symbol-prefix' % ns_map['c']])
+
+init_constants(girxml, settings)
+
+outfile.write("""<?xml version=\"1.0\"?>
+<!DOCTYPE nm-setting-docs [
+<!ENTITY quot "&#34;">
+]>
+<nm-setting-docs>
+""")
+
+for settingxml in settings:
+    if settingxml.attrib.has_key('abstract'):
+        continue
+
+    new_func = NM.__getattr__(settingxml.attrib['name'])
+    setting = new_func()
+
+    outfile.write("  <setting name=\"%s\">\n" % setting.props.name)
+
+    properties = sorted(GObject.list_properties(setting), key=lambda prop: prop.name)
+    for pspec in properties:
+        propxml = settingxml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
+        if propxml is None:
+            propxml = basexml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
+        if propxml is None:
+            propxml = ipxml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
+
+        value_type = get_prop_type(setting, pspec, propxml)
+        value_desc = get_docs(setting, pspec, propxml)
+        default_value = get_default_value(setting, pspec, propxml)
+
+        if default_value is not None:
+            outfile.write("    <property name=\"%s\" type=\"%s\" default=\"%s\" description=\"%s\" />\n" %
+                          (pspec.name, value_type, escape(default_value), escape(value_desc)))
+        else:
+            outfile.write("    <property name=\"%s\" type=\"%s\" description=\"%s\" />\n" %
+                          (pspec.name, value_type, escape(value_desc)))
+
+    outfile.write("  </setting>\n")
+
+outfile.write("</nm-setting-docs>\n")
+outfile.close()