qdoc: WebXML output format: Fix issues with example pages

Since QDoc no longer stores example files as nodes in its tree, the
WebXMLGenerator failed to generate output for those files.

Fix this by generating those pages explicitly as needed, overriding
functions from the Generator base class.

Prevent QDocIndexFiles from writing nested <page> elements when
WebXMLGenerator is in use, as that does not work as expected with
shiboken2.

Fixes: PYSIDE-1088
Change-Id: I01c2af2391726f448271fdb810ffc3da923caca5
Reviewed-by: Venugopal Shivashankar <Venugopal.Shivashankar@qt.io>
Reviewed-by: Paul Wicking <paul.wicking@qt.io>

5 files changed, 223 insertions, 83 deletions

diff --git a/src/qdoc/generator.cpp b/src/qdoc/generator.cpp
index de9976502..d5caa8bd0 100644
--- a/src/qdoc/generator.cpp
+++ b/src/qdoc/generator.cpp
@@ -447,9 +447,13 @@ QString Generator::fileBase(const Node *node) const
 
 /*!
   Constructs an href link from an example file name, which
-  is a path to the example file.
+  is a path to the example file. If \a fileExtension is
+  empty (default value), retrieve the file extension from
+  the generator.
  */
-QString Generator::linkForExampleFile(const QString &path, const Node *parent)
+QString Generator::linkForExampleFile(const QString &path,
+                                      const Node *parent,
+                                      const QString &fileExt)
 {
     QString link = path;
     QString modPrefix(parent->physicalModuleName());
@@ -460,11 +464,31 @@ QString Generator::linkForExampleFile(const QString &path, const Node *parent)
     QString res;
     transmogrify(link, res);
     res.append(QLatin1Char('.'));
-    res.append(fileExtension());
+    res.append(fileExt);
+    if (fileExt.isEmpty())
+        res.append(fileExtension());
     return res;
 }
 
 /*!
+    Helper function to construct a title for a file or image page
+    included in an example.
+*/
+QString Generator::exampleFileTitle(const ExampleNode *relative,
+                                    const QString &fileName)
+{
+    QString suffix;
+    if (relative->files().contains(fileName))
+        suffix = QLatin1String(" Example File");
+    else if (relative->images().contains(fileName))
+        suffix = QLatin1String(" Image File");
+    else
+        return suffix;
+
+    return fileName.mid(fileName.lastIndexOf(QLatin1Char('/')) + 1) + suffix;
+}
+
+/*!
   If the \a node has a URL, return the URL as the file name.
   Otherwise, construct the file name from the fileBase() and
   either the provided \a extension or fileExtension(), and
@@ -939,62 +963,75 @@ void Generator::generateBody(const Node *node, CodeMarker *marker)
             }
         }
     }
+    generateRequiredLinks(node, marker);
+}
+
+/*!
+  Generates either a link to the project folder for example \a node, or a list
+  of links files/images if 'url.examples config' variable is not defined.
+
+  Does nothing for non-example nodes.
+*/
+void Generator::generateRequiredLinks(const Node *node, CodeMarker *marker)
+{
+    if (!node->isExample())
+        return;
+
+    const ExampleNode *en = static_cast<const ExampleNode *>(node);
+    QString exampleUrl = config()->getString(CONFIG_URL + Config::dot + CONFIG_EXAMPLES);
 
-    // For examples, generate either a link to the project directory
-    // (if url.examples is defined), or a list of files/images.
-    if (node->isExample()) {
-        const ExampleNode* en = static_cast<const ExampleNode*>(node);
-        QString exampleUrl = config()->getString(CONFIG_URL + Config::dot + CONFIG_EXAMPLES);
-        if (!exampleUrl.isEmpty()) {
-            generateLinkToExample(en, marker, exampleUrl);
-        } else if (!en->noAutoList()) {
-            generateFileList(en, marker, false);
-            generateFileList(en, marker, true);
+    if (exampleUrl.isEmpty()) {
+        if (!en->noAutoList()) {
+            generateFileList(en, marker, false); // files
+            generateFileList(en, marker, true);  // images
         }
+    } else {
+        generateLinkToExample(en, marker, exampleUrl);
     }
 }
 
 /*!
-  Generates a link to the project folder for example node \a en.
-  \a baseUrl is the base URL - path information is available in
-  the example node's name() and 'examplesinstallpath' configuration
-  variable.
+  Generates an external link to the project folder for example \a node.
+  The path to the example is appended to \a baseUrl string, or to a
+  specific location within the string marked with the placeholder '\1'
+  character.
 */
 void Generator::generateLinkToExample(const ExampleNode *en,
                                       CodeMarker *marker,
                                       const QString &baseUrl)
 {
-        Text text;
-        QString exampleUrl(baseUrl);
-
-        if (!exampleUrl.contains("\1")) {
-            if (!exampleUrl.endsWith("/"))
-                exampleUrl += "/";
-            exampleUrl += "\1";
-        }
-
-        // Name of the example node is the path, relative to install path
-        QStringList path = QStringList()
-            << config()->getString(CONFIG_EXAMPLESINSTALLPATH)
-            << en->name();
-        path.removeAll({});
-
-        QString link;
+    QString exampleUrl(baseUrl);
+    QString link;
 #ifndef QT_BOOTSTRAPPED
-        link = QUrl(baseUrl).host();
+    link = QUrl(exampleUrl).host();
 #endif
-        if (!link.isEmpty())
-            link.prepend(" @ ");
-        link.prepend("Example project");
+    if (!link.isEmpty())
+        link.prepend(" @ ");
+    link.prepend("Example project");
+
+    const QLatin1Char separator('/');
+    const QLatin1Char placeholder('\1');
+    if (!exampleUrl.contains(placeholder)) {
+        if (!exampleUrl.endsWith(separator))
+            exampleUrl += separator;
+        exampleUrl += placeholder;
+    }
 
-        text << Atom::ParaLeft
-             << Atom(Atom::Link, exampleUrl.replace("\1", path.join("/")))
-             << Atom(Atom::FormattingLeft, ATOM_FORMATTING_LINK)
-             << Atom(Atom::String, link)
-             << Atom(Atom::FormattingRight, ATOM_FORMATTING_LINK)
-             << Atom::ParaRight;
+    // Construct a path to the example; <install path>/<example name>
+    QStringList path = QStringList()
+        << config()->getString(CONFIG_EXAMPLESINSTALLPATH)
+        << en->name();
+    path.removeAll({});
+
+    Text text;
+    text << Atom::ParaLeft
+         << Atom(Atom::Link, exampleUrl.replace(placeholder, path.join(separator)))
+         << Atom(Atom::FormattingLeft, ATOM_FORMATTING_LINK)
+         << Atom(Atom::String, link)
+         << Atom(Atom::FormattingRight, ATOM_FORMATTING_LINK)
+         << Atom::ParaRight;
 
-        generateText(text, 0, marker);
+    generateText(text, 0, marker);
 }
 
 /*!
diff --git a/src/qdoc/generator.h b/src/qdoc/generator.h
index d02c59f44..61478b2f6 100644
--- a/src/qdoc/generator.h
+++ b/src/qdoc/generator.h
@@ -70,9 +70,12 @@ public:
     virtual void terminateGenerator();
 
     QString fullDocumentLocation(const Node *node, bool useSubdir = false);
-    const Config* config() { return config_; }
-    QString linkForExampleFile(const QString &path, const Node *parent);
-
+    const Config *config() { return config_; }
+    QString linkForExampleFile(const QString &path,
+                               const Node *parent,
+                               const QString &fileExt = QString());
+    static QString exampleFileTitle(const ExampleNode *relative,
+                                    const QString &fileName);
     static Generator *currentGenerator() { return currentGenerator_; }
     static Generator *generatorForFormat(const QString& format);
     static void initialize(const Config& config);
@@ -155,11 +158,10 @@ protected:
                                  const Node *relative,
                                  CodeMarker *marker,
                                  bool generate,
-                                 int& numGeneratedAtoms);
-    void generateLinkToExample(const ExampleNode *en,
-                               CodeMarker *marker,
-                               const QString &baseUrl);
-    void generateFileList(const ExampleNode* en, CodeMarker* marker, bool images);
+                                 int &numGeneratedAtoms);
+    void generateRequiredLinks(const Node *node, CodeMarker *marker);
+    void generateLinkToExample(const ExampleNode *en, CodeMarker *marker, const QString &exampleUrl);
+    virtual void generateFileList(const ExampleNode *en, CodeMarker *marker, bool images);
     void generateSince(const Node *node, CodeMarker *marker);
     void generateStatus(const Node *node, CodeMarker *marker);
     void generatePrivateSignalNote(const Node* node, CodeMarker* marker);
diff --git a/src/qdoc/qdocindexfiles.cpp b/src/qdoc/qdocindexfiles.cpp
index b9347c108..a9f996a0f 100644
--- a/src/qdoc/qdocindexfiles.cpp
+++ b/src/qdoc/qdocindexfiles.cpp
@@ -1271,9 +1271,11 @@ bool QDocIndexFiles::generateIndexSection(QXmlStreamWriter &writer, Node *node,
             }
         }
     }
-    if (node->isExample()) {
-        const ExampleNode* en = static_cast<const ExampleNode*>(node);
-        foreach (const QString& file, en->files()) {
+    // WebXMLGenerator - skip the nested <page> elements for example
+    // files/images, as the generator produces them separately
+    if (node->isExample() && gen_->format() != QLatin1String("WebXML")) {
+        const ExampleNode *en = static_cast<const ExampleNode *>(node);
+        foreach (const QString &file, en->files()) {
             writer.writeStartElement("page");
             writer.writeAttribute("name", file);
             QString href = gen_->linkForExampleFile(file, en);
@@ -1281,11 +1283,11 @@ bool QDocIndexFiles::generateIndexSection(QXmlStreamWriter &writer, Node *node,
             writer.writeAttribute("status", "active");
             writer.writeAttribute("subtype", "file");
             writer.writeAttribute("title", "");
-            writer.writeAttribute("fulltitle", file.mid(file.lastIndexOf('/') + 1) + " Example File");
+            writer.writeAttribute("fulltitle", Generator::exampleFileTitle(en, file));
             writer.writeAttribute("subtitle", file);
             writer.writeEndElement(); // page
         }
-        foreach (const QString& file, en->images()) {
+        foreach (const QString &file, en->images()) {
             writer.writeStartElement("page");
             writer.writeAttribute("name", file);
             QString href = gen_->linkForExampleFile(file, en);
@@ -1293,7 +1295,7 @@ bool QDocIndexFiles::generateIndexSection(QXmlStreamWriter &writer, Node *node,
             writer.writeAttribute("status", "active");
             writer.writeAttribute("subtype", "image");
             writer.writeAttribute("title", "");
-            writer.writeAttribute("fulltitle", file.mid(file.lastIndexOf('/') + 1) + " Image File");
+            writer.writeAttribute("fulltitle", Generator::exampleFileTitle(en, file));
             writer.writeAttribute("subtitle", file);
             writer.writeEndElement(); // page
         }
diff --git a/src/qdoc/webxmlgenerator.cpp b/src/qdoc/webxmlgenerator.cpp
index 45178c3c1..20c655890 100644
--- a/src/qdoc/webxmlgenerator.cpp
+++ b/src/qdoc/webxmlgenerator.cpp
@@ -29,6 +29,7 @@
 #include "webxmlgenerator.h"
 #include "node.h"
 #include "separator.h"
+#include "quoter.h"
 #include "tree.h"
 #include "qdocdatabase.h"
 #include "helpprojectwriter.h"
@@ -64,10 +65,17 @@ QString WebXMLGenerator::fileExtension() const
     return "html";
 }
 
-int WebXMLGenerator::generateAtom(const Atom * /* atom, */,
-                                  const Node * /* relative */,
-                                  CodeMarker * /* marker */)
+/*!
+    Most of the output is generated by QDocIndexFiles and the append() callback.
+    Some pages produce supplementary output while being generated, and that's
+    handled here.
+*/
+int WebXMLGenerator::generateAtom(const Atom *atom,
+                                  const Node *relative,
+                                  CodeMarker *marker)
 {
+    if (supplement && currentWriter)
+        addAtomElements(*currentWriter.data(), atom, relative, marker);
     return 0;
 }
 
@@ -94,21 +102,64 @@ void WebXMLGenerator::generateCppReferencePage(Aggregate *aggregate, CodeMarker
 void WebXMLGenerator::generatePageNode(PageNode *pn, CodeMarker * /* marker */)
 {
     QByteArray data;
+    currentWriter.reset(new QXmlStreamWriter(&data));
+    currentWriter->setAutoFormatting(true);
+    beginSubPage(pn, Generator::fileName(pn, "webxml"));
+    currentWriter->writeStartDocument();
+    currentWriter->writeStartElement("WebXML");
+    currentWriter->writeStartElement("document");
+
+    generateIndexSections(*currentWriter.data(), pn);
+
+    currentWriter->writeEndElement(); // document
+    currentWriter->writeEndElement(); // WebXML
+    currentWriter->writeEndDocument();
+
+    out() << data;
+    endSubPage();
+}
+
+void WebXMLGenerator::generateExampleFilePage(const Node *en,
+                                            const QString &file,
+                                            CodeMarker * /* marker */)
+{
+    QByteArray data;
     QXmlStreamWriter writer(&data);
     writer.setAutoFormatting(true);
-    beginSubPage(pn, Generator::fileName(pn, "webxml"));
+    beginFilePage(en, linkForExampleFile(file, en, "webxml"));
     writer.writeStartDocument();
     writer.writeStartElement("WebXML");
     writer.writeStartElement("document");
+    writer.writeStartElement("page");
+    writer.writeAttribute("name", file);
+    writer.writeAttribute("href", linkForExampleFile(file, en));
+    QString title = exampleFileTitle(static_cast<const ExampleNode *>(en), file);
+    writer.writeAttribute("title", title);
+    writer.writeAttribute("fulltitle", title);
+    writer.writeAttribute("subtitle", file);
+    writer.writeStartElement("description");
+    QString userFriendlyFilePath; // unused
+    writer.writeAttribute("path", Doc::resolveFile(en->doc().location(),
+                                                   file,
+                                                   &userFriendlyFilePath));
+    writer.writeAttribute("line", "0");
+    writer.writeAttribute("column", "0");
+
+    Quoter quoter;
+    Doc::quoteFromFile(en->doc().location(), quoter, file);
+    QString code = quoter.quoteTo(en->location(), QString(), QString());
+    writer.writeTextElement("code", trimmedTrailing(code, QString(), QString()));
 
-    generateIndexSections(writer, pn);
 
+    writer.writeEndElement(); // description
+    writer.writeEndElement(); // page
     writer.writeEndElement(); // document
     writer.writeEndElement(); // WebXML
     writer.writeEndDocument();
 
+
     out() << data;
-    endSubPage();
+    endFilePage();
 }
 
 void WebXMLGenerator::generateIndexSections(QXmlStreamWriter &writer, Node *node)
@@ -179,6 +230,12 @@ void WebXMLGenerator::append(QXmlStreamWriter &writer, Node *node)
         }
         writer.writeEndElement(); // see-also
     }
+    if (node->isExample()) {
+        supplement = true;
+        generateRequiredLinks(node, marker_);
+        supplement = false;
+    }
+
     writer.writeEndElement(); // description
 }
 
@@ -348,6 +405,26 @@ const Atom *WebXMLGenerator::addAtomElements(QXmlStreamWriter &writer,
         }
         break;
 
+    case Atom::ExampleFileLink:
+    {
+        if (!inLink) {
+            QString link = linkForExampleFile(atom->string(), relative);
+            if (!link.isEmpty())
+                startLink(writer, atom, relative, link);
+        }
+    }
+    break;
+
+    case Atom::ExampleImageLink:
+    {
+        if (!inLink) {
+            QString link = atom->string();
+            if (!link.isEmpty())
+                startLink(writer, atom, nullptr, "images/used-in-examples/" + link);
+        }
+    }
+    break;
+
     case Atom::FootnoteLeft:
         writer.writeStartElement("footnote");
         break;
@@ -452,7 +529,7 @@ const Atom *WebXMLGenerator::addAtomElements(QXmlStreamWriter &writer,
         if (!inLink) {
             const Node *node = nullptr;
             QString link = getLink(atom, relative, &node);
-            if (node)
+            if (!link.isEmpty())
                 startLink(writer, atom, node, link);
         }
         break;
@@ -667,27 +744,41 @@ const Atom *WebXMLGenerator::addAtomElements(QXmlStreamWriter &writer,
 void WebXMLGenerator::startLink(QXmlStreamWriter &writer, const Atom *atom,
                                 const Node *node, const QString &link)
 {
-    QString fullName = node->fullName();
+    QString fullName = link;
+    if (node)
+        fullName = node->fullName();
     if (!fullName.isEmpty() && !link.isEmpty()) {
         writer.writeStartElement("link");
         writer.writeAttribute("raw", atom->string());
         writer.writeAttribute("href", link);
         writer.writeAttribute("type", targetType(node));
-        switch (node->nodeType()) {
-        case Node::Enum:
-            writer.writeAttribute("enum", fullName);
-            break;
-        case Node::Page:
-            writer.writeAttribute("page", fullName);
-            break;
-        case Node::Property:
-        {
-            const PropertyNode *propertyNode = static_cast<const PropertyNode *>(node);
-            if (propertyNode->getters().size() > 0)
-                writer.writeAttribute("getter", propertyNode->getters().at(0)->fullName());
-        }
-        default:
-            ;
+        if (node) {
+            switch (node->nodeType()) {
+            case Node::Enum:
+                writer.writeAttribute("enum", fullName);
+                break;
+            case Node::Example:
+            {
+                const ExampleNode *en = static_cast<const ExampleNode *>(node);
+                QString fileTitle = exampleFileTitle(en, atom->string());
+                if (!fileTitle.isEmpty()) {
+                    writer.writeAttribute("page", fileTitle);
+                    break;
+                }
+            }
+            // fall through
+            case Node::Page:
+                writer.writeAttribute("page", fullName);
+                break;
+            case Node::Property:
+            {
+                const PropertyNode *propertyNode = static_cast<const PropertyNode *>(node);
+                if (propertyNode->getters().size() > 0)
+                    writer.writeAttribute("getter", propertyNode->getters().at(0)->fullName());
+            }
+            default:
+                ;
+            }
         }
         inLink = true;
     }
@@ -695,6 +786,9 @@ void WebXMLGenerator::startLink(QXmlStreamWriter &writer, const Atom *atom,
 
 QString WebXMLGenerator::targetType(const Node *node)
 {
+    if (!node)
+        return "external";
+
     switch (node->nodeType()) {
     case Node::Namespace:
         return "namespace";
@@ -703,6 +797,7 @@ QString WebXMLGenerator::targetType(const Node *node)
     case Node::Union:
         return "class";
     case Node::Page:
+    case Node::Example:
         return "page";
     case Node::Enum:
         return "enum";
diff --git a/src/qdoc/webxmlgenerator.h b/src/qdoc/webxmlgenerator.h
index f243a5402..bdd28dd2f 100644
--- a/src/qdoc/webxmlgenerator.h
+++ b/src/qdoc/webxmlgenerator.h
@@ -29,13 +29,14 @@
 #ifndef WEBXMLGENERATOR_H
 #define WEBXMLGENERATOR_H
 
-#include <QtCore/qxmlstream.h>
-
 #include "codemarker.h"
 #include "config.h"
 #include "htmlgenerator.h"
 #include "qdocindexfiles.h"
 
+#include <QtCore/qxmlstream.h>
+#include <QtCore/qscopedpointer.h>
+
 QT_BEGIN_NAMESPACE
 
 class WebXMLGenerator : public HtmlGenerator, public IndexSectionWriter
@@ -55,6 +56,7 @@ protected:
     void generateCppReferencePage(Aggregate *aggregate, CodeMarker *marker) override;
     void generatePageNode(PageNode *pn, CodeMarker *marker) override;
     void generateDocumentation(Node *node) override;
+    void generateExampleFilePage(const Node *en, const QString &file, CodeMarker *marker) override;
     QString fileExtension() const override;
 
     virtual const Atom *addAtomElements(QXmlStreamWriter &writer, const Atom *atom,
@@ -78,6 +80,8 @@ private:
     bool hasQuotingInformation;
     int numTableRows;
     QString quoteCommand;
+    QScopedPointer<QXmlStreamWriter> currentWriter;
+    bool supplement = false;
 };
 
 QT_END_NAMESPACE