improved comment formatting for Python

10 files changed, 215 insertions, 142 deletions

diff --git a/Examples/test-suite/doxygen_basic_translate.i b/Examples/test-suite/doxygen_basic_translate.i
index e8c21f452..02f5a1c14 100644
--- a/Examples/test-suite/doxygen_basic_translate.i
+++ b/Examples/test-suite/doxygen_basic_translate.i
@@ -8,9 +8,12 @@
  * \brief
  * Brief description.
  * 
- * The comment text
+ * The comment text.
+ *
  * \author Some author
+ *
  * \return Some number
+ *
  * \sa function2
  */
 int function()
@@ -51,7 +54,6 @@ int function3(int a, int b)
  * \arg it's null
  *
  * \warning This may not work as expected
- *
  * \code
  * int main() { while(true); }
  * \endcode
diff --git a/Examples/test-suite/python/commentVerifier.py b/Examples/test-suite/python/commentVerifier.py
new file mode 100644
index 000000000..8a73b1950
--- /dev/null
+++ b/Examples/test-suite/python/commentVerifier.py
@@ -0,0 +1,24 @@
+
+def check(got, expected):
+  #if got is Null
+  #  raise RuntimeError('Expected comment string\n')
+  #gotStr = string.replace(got, ' ', '')
+  #gotStr = string.replace(gotStr, '\n', '')
+  # gotStr = string.replace(gotStr, '\t', '')
+  #expectedStr = string.replace(expected, ' ', '')
+  #expectedStr = string.replace(expectedStr, '\n', '')
+  #expectedStr = string.replace(expectedStr, '\t', '')
+  if not got == expected:
+    print "\n\n////////////////////////////////////////////////////////////////////////"
+    expectedFileName = "expected.txt"
+    gotFileName = "got.txt"
+    print "Output is also saved to files '" + expectedFileName + \
+                                   "' and '" + gotFileName + "'";
+
+    expectedFile = open(expectedFileName, "w")
+    expectedFile.write(expected)
+    expectedFile.close()
+    gotFile = open(gotFileName, "w")
+    gotFile.write(got)
+    gotFile.close()    
+    raise RuntimeError("Expected: [" + str(expected) + "]\n" + "Got     : [" + str(got) + "]\n")
diff --git a/Examples/test-suite/python/doxygen_basic_translate_runme.py b/Examples/test-suite/python/doxygen_basic_translate_runme.py
index 4c1376f44..36ebea5d0 100644..100755
--- a/Examples/test-suite/python/doxygen_basic_translate_runme.py
+++ b/Examples/test-suite/python/doxygen_basic_translate_runme.py
@@ -3,71 +3,77 @@
 import doxygen_basic_translate
 import string
 import sys
+import commentVerifier
 
-def check(got, expected):
-  #if got is Null
-  #  raise RuntimeError('Expected comment string\n')
-  gotStr = string.replace(got, ' ', '')
-  gotStr = string.replace(gotStr, '\n', '')
-  gotStr = string.replace(gotStr, '\t', '')
-  expectedStr = string.replace(expected, ' ', '')
-  expectedStr = string.replace(expectedStr, '\n', '')
-  expectedStr = string.replace(expectedStr, '\t', '')
-  if not gotStr == expectedStr:
-    raise RuntimeError("Expected: [" + str(expected) + "]\n" + "Got     : [" + str(got) + "]\n")
+commentVerifier.check(doxygen_basic_translate.function.__doc__,
+  """
+  Brief description.
 
-check(doxygen_basic_translate.function.__doc__, ''
-'  Brief description.\n'
-'  The comment text\n'
-'  Authors:\n'
-'  Some author\n'
-'  Return:\n'
-'  Some number\n'
-'  See also:\n'
-'  function2\n'
+  The comment text.
+
+  Author: Some author
+
+  Return: Some number
+
+  See also: function2
+  """
 )
-check(doxygen_basic_translate.function2.__doc__, ''
-'  A test of a very very very very very very very very very very very'
-'  very very very very very'
-'  very very very very very long comment'
-'  string.'
+commentVerifier.check(doxygen_basic_translate.function2.__doc__,
+  """
+  A test of a very very very very very very very very very very very very very very very very
+  very very very very very long comment string.
+  """
 )
-check(doxygen_basic_translate.function3.__doc__, ''
-'  ----------------------------------------------------------------\n'
-'  Overload 1:\n'
-'  ----------------------------------------------------------------\n'
-'  A test for overloaded functions\n'
-'  This is function __one__\n'
-'  ----------------------------------------------------------------\n'
-'  Overload 2:\n'
-'  ----------------------------------------------------------------\n'
-'  A test for overloaded functions\n'
-'  This is function __two__\n'
+commentVerifier.check(doxygen_basic_translate.function3.__doc__,
+  """
+  ----------------------------------------------------------------
+  Overload 1:
+  ----------------------------------------------------------------
+  A test for overloaded functions
+  This is function __one__
+
+  ----------------------------------------------------------------
+  Overload 2:
+  ----------------------------------------------------------------
+  A test for overloaded functions
+  This is function __two__
+
+  """
 )
-check(doxygen_basic_translate.function4.__doc__, ''
-'  A test of some mixed tag usage\n' 
-'  If: CONDITION {\n'
-'  This _code_fragment shows us something .\n'
-'  Title: Minuses:\n'
-'   -it\'s senseless\n'
-'   -it\'s stupid\n'
-'   -it\'s null\n'
-'  Warning:\n'
-'  This may not work as expected\n'
-'  int main() { while(true); }\n'
-'  }'
+commentVerifier.check(doxygen_basic_translate.function4.__doc__,
+  """
+  A test of some mixed tag usage
+  If: CONDITION {
+  This _code_ fragment shows us something .
+  Title: Minuses:
+   -it\'s senseless
+   -it\'s stupid
+   -it\'s null
+
+  Warning: This may not work as expected
+
+  int main() { while(true); }
+
+  }
+  """
 )
-check(doxygen_basic_translate.function5.__doc__, ''
-'  This is a post comment. \n'
+commentVerifier.check(doxygen_basic_translate.function5.__doc__,
+  """
+  This is a post comment. 
+  """
 )
-check(doxygen_basic_translate.function6.__doc__, ''
-'  Test for default args \n'
-'  Arguments: \n'
-'  a (int) -- Some parameter, default is 42\n'
+commentVerifier.check(doxygen_basic_translate.function6.__doc__,
+  """
+  Test for default args
+  Arguments:
+    a (int) -- Some parameter, default is 42
+  """
 )
-check(doxygen_basic_translate.function7.__doc__, ''
-' Test for a parameter with difficult type \n'
-'  (mostly for python) \n'
-'  Arguments: \n'
-'  a (Shape::superType *[10])                              -- Very strange param \n'
+commentVerifier.check(doxygen_basic_translate.function7.__doc__,
+  """
+  Test for a parameter with difficult type
+  (mostly for python)
+  Arguments:
+    a (Shape::superType *[10]) -- Very strange param
+  """
 )
diff --git a/Examples/test-suite/python/doxygen_misc_constructs_runme.py b/Examples/test-suite/python/doxygen_misc_constructs_runme.py
index 156a279b7..55b4a7ef6 100644..100755
--- a/Examples/test-suite/python/doxygen_misc_constructs_runme.py
+++ b/Examples/test-suite/python/doxygen_misc_constructs_runme.py
@@ -1,54 +1,48 @@
 #!/usr/bin/python
 
-import doxygen_tricky_constructs
+import doxygen_misc_constructs
 import string
 import sys
+import commentVerifier
 
-def check(got, expected):
-  #if got is Null
-  #  raise RuntimeError('Expected comment string\n')
-  gotStr = string.replace(got, ' ', '')
-  gotStr = string.replace(gotStr, '\n', '')
-  gotStr = string.replace(gotStr, '\t', '')
-  expectedStr = string.replace(expected, ' ', '')
-  expectedStr = string.replace(expectedStr, '\n', '')
-  expectedStr = string.replace(expectedStr, '\t', '')
-  if not gotStr == expectedStr:
-    raise RuntimeError("Expected: [" + str(expected) + "]\n" + "Got     : [" + str(got) + "]\n")
-
-check(doxygen_tricky_constructs.getAddress.__doc__, ''
-'Returns address of file line.'
-''
-'   Arguments:'
-'  fileName (int &)                              -- name of the file, where the source'
-'                                line is located'
-'  line (int)                              -- line number'
-'  isGetSize (bool)                              -- if set, for every object location'
-'                                both address and size are returned'
-''
-''
-'  Connection::getId() <br>'
-)
-check(doxygen_tricky_constructs.CConnectionConfig.__doc__, ''
-'    This class contains information for connection to winIDEA. Its methods'
-'    return reference to self, so we can use it like this:'
-'    <pre>'
-'    CConnectionConfig config = new CConnectionConfig();'
-'    config.discoveryPort(5534).dllPath("C: \yWinIDEA \onnect.dll").id("main");'
-'    </pre>'
-''
-'     All parameters are optional. Set only what is required, default values are'
-'    used for unspecified parameters.'
-'    <p>'
-''
-'     advancedWinIDEALaunching.py Python example. <br>'
-''
+
+commentVerifier.check(doxygen_misc_constructs.getAddress.__doc__,
+  r"""
+  Returns address of file line.
+
+  Arguments:
+    fileName (int &) -- name of the file, where the source line is located
+    line (int) -- line number
+    isGetSize (bool) -- if set, for every object location both address and size are returned
+
+  Connection::getId() 
+  """)
+
+commentVerifier.check(doxygen_misc_constructs.CConnectionConfig.__doc__,
+    r"""
+    This class contains information for connection to winIDEA. Its methods 
+    return reference to self, so we can use it like this:
+
+    CConnectionConfig config = new CConnectionConfig();
+    config.discoveryPort(5534).dllPath("C:\\myWinIDEA\\connect.dll").id("main");
+
+
+    All parameters are optional. Set only what is required, default values are
+    used for unspecified parameters.
+
+
+    advancedWinIDEALaunching.py Python example. 
+    """)
+
+commentVerifier.check(doxygen_misc_constructs.waitTime.__doc__,
+  r"""
+  Determines how long the 'isystem.connect' should wait for running 
+  instances to respond. Only one of 'lfWaitXXX' flags from IConnect::ELaunchFlags
+  may be specified.
+  """
 )
-check(doxygen_tricky_constructs.waitTime.__doc__, ''
-'  Determines how long the isystem.connect should wait for running'
-'  instances to respond. Only one of lfWaitXXX flags from IConnect::ELaunchFlags'
-'  may be specified.'
+commentVerifier.check(doxygen_misc_constructs.getConnection.__doc__,
+  r"""
+  This function returns connection id.
+  """
 )
-check(doxygen_tricky_constructs.getConnection.__doc__, ''
-'This class manages connection.'
-)
\ No newline at end of file
diff --git a/Examples/test-suite/python/doxygen_parsing_runme.py b/Examples/test-suite/python/doxygen_parsing_runme.py
index cb22bb369..cb22bb369 100644..100755
--- a/Examples/test-suite/python/doxygen_parsing_runme.py
+++ b/Examples/test-suite/python/doxygen_parsing_runme.py
diff --git a/Examples/test-suite/python/doxygen_translate_all_tags_runme.py b/Examples/test-suite/python/doxygen_translate_all_tags_runme.py
index f71cb7e11..f71cb7e11 100644..100755
--- a/Examples/test-suite/python/doxygen_translate_all_tags_runme.py
+++ b/Examples/test-suite/python/doxygen_translate_all_tags_runme.py
diff --git a/Examples/test-suite/python/doxygen_translate_runme.py b/Examples/test-suite/python/doxygen_translate_runme.py
index e7d819199..e7d819199 100644..100755
--- a/Examples/test-suite/python/doxygen_translate_runme.py
+++ b/Examples/test-suite/python/doxygen_translate_runme.py
diff --git a/Source/DoxygenTranslator/src/DoxygenParser.cpp b/Source/DoxygenTranslator/src/DoxygenParser.cpp
index 846da261b..9c07eb4cf 100644
--- a/Source/DoxygenTranslator/src/DoxygenParser.cpp
+++ b/Source/DoxygenTranslator/src/DoxygenParser.cpp
@@ -1247,6 +1247,16 @@ void DoxygenParser::tokenizeDoxygenComment(const std::string &doxygenComment,
 
   StringVector lines = split(doxygenComment, '\n');
 
+  // remove trailing spaces, because they cause additional new line at the end
+  // comment, which is wrong, because these spaces are space preceding
+  // end of comment :  '  */'
+  if (!doxygenComment.empty()  &&  doxygenComment[doxygenComment.size() - 1] == ' ') {
+    string lastLine = lines[lines.size() - 1];
+    if (trim(lastLine).empty()) {
+      lines.pop_back(); // remove trailing empy line
+    }
+  }
+
   for (StringVectorCIt it = lines.begin(); it != lines.end(); it++) {
     const string &line = *it;
     size_t pos = line.find_first_not_of(" \t");
diff --git a/Source/DoxygenTranslator/src/PyDocConverter.cpp b/Source/DoxygenTranslator/src/PyDocConverter.cpp
index 4f3566635..675f2ae11 100644
--- a/Source/DoxygenTranslator/src/PyDocConverter.cpp
+++ b/Source/DoxygenTranslator/src/PyDocConverter.cpp
@@ -31,32 +31,33 @@ void PyDocConverter::fillStaticTables() {
 
   // table of section titles, they are printed only once
   // for each group of specified doxygen commands
-  sectionTitles["author"] = "Authors:";
-  sectionTitles["authors"] = "Authors:";
-  sectionTitles["copyright"] = "Copyright:";
-  sectionTitles["deprecated"] = "Deprecated:";
-  sectionTitles["example"] = "Example:";
-  sectionTitles["exception"] = "Throws:";
-  sectionTitles["param"] = "Arguments:";
-  sectionTitles["tparam"] = "Arguments:";
-  sectionTitles["note"] = "Notes:";
-  sectionTitles["remark"] = "Remarks:";
-  sectionTitles["remarks"] = "Remarks:";
-  sectionTitles["warning"] = "Warning:";
-  sectionTitles["result"] = "Return:";
-  sectionTitles["return"] = "Return:";
-  sectionTitles["returns"] = "Return:";
-  sectionTitles["sa"] = "See also:";
-  sectionTitles["see"] = "See also:";
-  sectionTitles["since"] = "Since:";
-  sectionTitles["throw"] = "Throws:";
-  sectionTitles["throws"] = "Throws:";
-  sectionTitles["todo"] = "TODO:";
-  sectionTitles["version"] = "Version:";
-
-  // these commands insert HTML tags
+  sectionTitles["author"] = "Author: ";
+  sectionTitles["authors"] = "Authors: ";
+  sectionTitles["copyright"] = "Copyright: ";
+  sectionTitles["deprecated"] = "Deprecated: ";
+  sectionTitles["example"] = "Example: ";
+  sectionTitles["exception"] = "Throws: ";
+  sectionTitles["param"] = "Arguments:\n";
+  sectionTitles["tparam"] = "Arguments:\n";
+  sectionTitles["note"] = "Notes: ";
+  sectionTitles["remark"] = "Remarks: ";
+  sectionTitles["remarks"] = "Remarks: ";
+  sectionTitles["warning"] = "Warning: ";
+  sectionTitles["result"] = "Return: ";
+  sectionTitles["return"] = "Return: ";
+  sectionTitles["returns"] = "Returns: ";
+  sectionTitles["sa"] = "See also: ";
+  sectionTitles["see"] = "See also: ";
+  sectionTitles["since"] = "Since: ";
+  sectionTitles["throw"] = "Throw: ";
+  sectionTitles["throws"] = "Throws: ";
+  sectionTitles["todo"] = "TODO: ";
+  sectionTitles["version"] = "Version: ";
+
   tagHandlers["a"] = make_pair(&PyDocConverter::handleTagWrap, "_");
   tagHandlers["b"] = make_pair(&PyDocConverter::handleTagWrap, "__");
+  // \c command is translated as single quotes around next word
+  tagHandlers["c"] = make_pair(&PyDocConverter::handleTagWrap, "'");
   tagHandlers["cite"] = make_pair(&PyDocConverter::handleTagWrap, "'");
   tagHandlers["e"] = make_pair(&PyDocConverter::handleTagWrap, "_");
   // these commands insert just a single char, some of them need to be escaped
@@ -78,7 +79,6 @@ void PyDocConverter::fillStaticTables() {
   tagHandlers["authors"] = make_pair(&PyDocConverter::handleParagraph, "");
   tagHandlers["brief"] = make_pair(&PyDocConverter::handleParagraph, "");
   tagHandlers["bug"] = make_pair(&PyDocConverter::handleParagraph, "");
-  tagHandlers["c"] = make_pair(&PyDocConverter::handleParagraph, " ");
   tagHandlers["code"] = make_pair(&PyDocConverter::handleParagraph, "");
   tagHandlers["copyright"] = make_pair(&PyDocConverter::handleParagraph, "");
   tagHandlers["date"] = make_pair(&PyDocConverter::handleParagraph, "");
@@ -196,7 +196,7 @@ std::string PyDocConverter::translateSubtree(DoxygenEntity & doxygenEntity) {
     if (it != sectionTitles.end()) {
       if (it->second != currentSection) {
         currentSection = it->second;
-        translatedComment += currentSection + "\n";
+        translatedComment += currentSection;
       }
     }
     translateEntity(*p, translatedComment);
@@ -215,22 +215,31 @@ void PyDocConverter::translateEntity(DoxygenEntity & doxyEntity, std::string &tr
     (this->*(it->second.first))(doxyEntity, translatedComment, it->second.second);
 }
 
+
 void PyDocConverter::handleParagraph(DoxygenEntity& tag, std::string& translatedComment, std::string &arg) {
   translatedComment += translateSubtree(tag) + arg;
 }
+
+
 void PyDocConverter::handlePlainString(DoxygenEntity& tag, std::string& translatedComment, std::string&) {
   translatedComment += tag.data;
   if (tag.data.size() && tag.data[tag.data.size()-1] != ' ')
     translatedComment += "";
 }
+
+
 void PyDocConverter::handleTagMessage(DoxygenEntity& tag, std::string& translatedComment, std::string &arg) {
   std::string dummy;
   translatedComment += arg;
   handleParagraph(tag, translatedComment, dummy);
 }
+
+
 void PyDocConverter::handleTagChar(DoxygenEntity& tag, std::string& translatedComment, std::string&) {
   translatedComment += tag.typeOfEntity;
 }
+
+
 void PyDocConverter::handleTagIf(DoxygenEntity& tag, std::string& translatedComment, std::string &arg) {
   std::string dummy;
   translatedComment += arg;
@@ -240,6 +249,8 @@ void PyDocConverter::handleTagIf(DoxygenEntity& tag, std::string& translatedComm
     translatedComment += " {" + translateSubtree(tag) + "}";
   }
 }
+
+
 void PyDocConverter::handleTagPar(DoxygenEntity& tag, std::string& translatedComment, std::string&) {
   std::string dummy;
   translatedComment += "Title: ";
@@ -248,6 +259,8 @@ void PyDocConverter::handleTagPar(DoxygenEntity& tag, std::string& translatedCom
   tag.entityList.pop_front();
   handleParagraph(tag, translatedComment, dummy);
 }
+
+
 void PyDocConverter::handleTagImage(DoxygenEntity& tag, std::string& translatedComment, std::string&) {
   if (tag.entityList.size() < 2)
     return;
@@ -258,7 +271,11 @@ void PyDocConverter::handleTagImage(DoxygenEntity& tag, std::string& translatedC
   if (tag.entityList.size())
     translatedComment += "(" + tag.entityList.begin()->data + ")";
 }
-void PyDocConverter::handleTagParam(DoxygenEntity& tag, std::string& translatedComment, std::string&) {
+
+
+void PyDocConverter::handleTagParam(DoxygenEntity& tag,
+                                         std::string& translatedComment,
+                                         std::string&) {
   std::string dummy;
   if (tag.entityList.size() < 2)
     return;
@@ -270,9 +287,11 @@ void PyDocConverter::handleTagParam(DoxygenEntity& tag, std::string& translatedC
   if (!paramType.size())
     paramType = "none";
 
-  translatedComment += paramNameEntity.data + " (" + paramType + ") -- ";
+  translatedComment += "  " + paramNameEntity.data + " (" + paramType + ") --";
   handleParagraph(tag, translatedComment, dummy);
 }
+
+
 void PyDocConverter::handleTagWrap(DoxygenEntity& tag, std::string& translatedComment, std::string &arg) {
   if (tag.entityList.size()) { // do not include empty tags
     std::string tagData = translateSubtree(tag);
@@ -281,13 +300,16 @@ void PyDocConverter::handleTagWrap(DoxygenEntity& tag, std::string& translatedCo
     if (wsPos != std::string::npos && wsPos != tagData.size() - 1)
       translatedComment += arg + tagData.substr(0, wsPos + 1) + arg + tagData.substr(wsPos + 1);
     else
-      translatedComment += arg + tagData + arg + " ";
+      translatedComment += arg + tagData + arg;
   }
 }
+
+
 void PyDocConverter::handleNewLine(DoxygenEntity&, std::string& translatedComment, std::string&) {
   translatedComment += "\n";
 }
 
+
 String *PyDocConverter::makeDocumentation(Node *n) {
   String *documentation;
   std::string pyDocString, result;
@@ -358,7 +380,13 @@ String *PyDocConverter::makeDocumentation(Node *n) {
   }
 
   // if we got something log the result and construct DOH string to return
-  if (pyDocString.length()) {
+  if (!pyDocString.empty()) {
+
+    // remove the last '\n' since additional one is added during writing to file
+    if (pyDocString[pyDocString.size() - 1] == '\n') {
+      pyDocString.erase(pyDocString.size() - 1);
+    }
+
     result = pyDocString;
 
     if (debug) {
diff --git a/Source/Modules/python.cxx b/Source/Modules/python.cxx
index 1d2096c81..f3f1df832 100644
--- a/Source/Modules/python.cxx
+++ b/Source/Modules/python.cxx
@@ -1257,26 +1257,35 @@ public:
     //
     // otherwise, put it all on a single line
     //
+    // All comments translated from doxygen are given as raw stringr (prefix "r"),
+    // because '\' is used often in comments, but may break Python module from
+    // loading. For example, in doxy comment one may write path in quotes:
+    //
+    //     This is path to file "C:\x\file.txt"
+    //
+    // Python will no load the module with such comment becaue of illegal
+    // escape '\x'. '\' may additionally appear in verbatim or htmlonly sections
+    // of doxygen doc, Latex expressions, ...
     if (have_auto && have_ds) {	// Both autodoc and docstring are present
       doc = NewString("");
-      Printv(doc, triple_double, "\n", pythoncode(autodoc, indent), "\n", pythoncode(str, indent), indent, triple_double, NIL);
+      Printv(doc, "r", triple_double, "\n", pythoncode(autodoc, indent), "\n", pythoncode(str, indent), indent, triple_double, NIL);
     } else if (!have_auto && have_ds) {	// only docstring
       if (Strchr(str, '\n') == 0) {
 	doc = NewStringf("%s%s%s", triple_double, str, triple_double);
       } else {
 	doc = NewString("");
-	Printv(doc, triple_double, "\n", pythoncode(str, indent), indent, triple_double, NIL);
+	Printv(doc, "r", triple_double, "\n", pythoncode(str, indent), indent, triple_double, NIL);
       }
     } else if (have_auto && !have_ds) {	// only autodoc
       if (Strchr(autodoc, '\n') == 0) {
 	doc = NewStringf("%s%s%s", triple_double, autodoc, triple_double);
       } else {
 	doc = NewString("");
-	Printv(doc, triple_double, "\n", pythoncode(autodoc, indent), indent, triple_double, NIL);
+	Printv(doc, "r", triple_double, "\n", pythoncode(autodoc, indent), indent, triple_double, NIL);
       }
     } else if (have_doxygen) { // the lowest priority
       doc = NewString("");
-      Printv(doc, triple_double, "\n", pythoncode(doxygen_comment, indent), indent, triple_double, NIL);
+      Printv(doc, "r", triple_double, "\n", pythoncode(doxygen_comment, indent), indent, triple_double, NIL);
     }
     else
       doc = NewString("");