improved handling of word commands if puntuation is following the word, preserved spaces at start of comment line

8 files changed, 223 insertions, 129 deletions

diff --git a/Examples/test-suite/doxygen_misc_constructs.h b/Examples/test-suite/doxygen_misc_constructs.h
index edbea3ed5..98856ac9a 100644
--- a/Examples/test-suite/doxygen_misc_constructs.h
+++ b/Examples/test-suite/doxygen_misc_constructs.h
@@ -69,6 +69,26 @@ void backslashC()
  * <pre>

  * ['retVal < 10', 'g_counter == 23 && g_mode & 3']

  *</pre>

+ *

+ * Both words should be emphasized \b isystem.connect.

+ * But not the last period. For \b example, comma should not be emphasized.

+ * Similar \b for: double colon.

+ *

+ * Spaces at the start of line should be taken into account:

+ * @param id used as prefix in log

+ *     statements. The default value is empty string, which is OK if

+ *     there is only one app. instance. Example:

+ *     <pre>

+ *         ctrl.setBP("func1");

+ *     </pre>

+ *     If we set the id to \c main_, we get:

+ *     <pre>

+ *         main_ctrl.setBP("func1");

+ *     </pre>

+ *

+ * @param fileName name of the log file

  */

-void cycle()

+void cycle(int id, char *fileName)

 {}

+

+

diff --git a/Examples/test-suite/java/doxygen_misc_constructs_runme.java b/Examples/test-suite/java/doxygen_misc_constructs_runme.java
index b14918a64..ba0f2b265 100644
--- a/Examples/test-suite/java/doxygen_misc_constructs_runme.java
+++ b/Examples/test-suite/java/doxygen_misc_constructs_runme.java
@@ -154,13 +154,29 @@ public class doxygen_misc_constructs_runme {
                 " @see MyClass#fun(char,float)\n" +
     		"");
 
-   wantedComments.put("doxygen_misc_constructs.doxygen_misc_constructs.cycle()",
+   wantedComments.put("doxygen_misc_constructs.doxygen_misc_constructs.cycle(int, java.lang.String)",
     		" The next line contains expression:\n" +
                 " <pre>\n" +
     		" ['retVal &lt; 10', 'g_counter == 23 &amp;&amp; g_mode &amp; 3']\n" +
                 " </pre>\n" +
     		"\n" +
-    		"");
+                " Both words should be emphasized <b>isystem.connect</b>.\n" +
+                " But not the last period. For <b>example</b>, comma should not be emphasized.\n" +
+                " Similar <b>for</b>: double colon.\n" +
+    		"\n" +
+                " Spaces at the start of line should be taken into account:\n" +
+                " @param id used as prefix in log\n" +
+                "     statements. The default value is empty string, which is OK if\n" +
+                "     there is only one app. instance. Example:\n" +
+                "     <pre>\n" +
+                "         ctrl.setBP(\"func1\");\n" +
+                "     </pre>\n" +
+                "     If we set the id to <code>main_</code>, we get:\n" +
+                "     <pre>\n" +
+                "         main_ctrl.setBP(\"func1\");\n" +
+                "     </pre>\n" +
+                "\n" +
+                " @param fileName name of the log file\n");
  
     
     // and ask the parser to check comments for us
diff --git a/Examples/test-suite/java/doxygen_translate_all_tags_runme.java b/Examples/test-suite/java/doxygen_translate_all_tags_runme.java
index 69a1169ca..5374d97c4 100644
--- a/Examples/test-suite/java/doxygen_translate_all_tags_runme.java
+++ b/Examples/test-suite/java/doxygen_translate_all_tags_runme.java
@@ -52,7 +52,7 @@ public class doxygen_translate_all_tags_runme {
     		" and detailed description of some thing \n");
     
     wantedComments.put("doxygen_translate_all_tags.doxygen_translate_all_tags.func03(int)",
-                       " Comment for <b>func03().</b>\n" +
+                       " Comment for <b>func03()</b>.\n" +
                        " <i>italicword </i>\n" +
     		" <i>emphazedWord </i>\n" +
                 " @ example someFile.txt\n" +
@@ -87,7 +87,7 @@ public class doxygen_translate_all_tags_runme {
     		" describing invariant. \n");
     
     wantedComments.put("doxygen_translate_all_tags.doxygen_translate_all_tags.func06(int)",
-                       " Comment for <b>func06().</b>\n" +
+                       " Comment for <b>func06()</b>.\n" +
                        " This will only appear in LATeX \n" +
     		" <ul> \n" +
     		" <li>Some unordered list \n" +
@@ -98,7 +98,7 @@ public class doxygen_translate_all_tags_runme {
     		" This will only appear in man\n");
     
     wantedComments.put("doxygen_translate_all_tags.doxygen_translate_all_tags.func07(int)",
-                       " Comment for <b>func07().</b>\n" +
+                       " Comment for <b>func07()</b>.\n" +
                        " Note: Here \n" +
     		" is the note! \n" +
     		" This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.\n" +
diff --git a/Examples/test-suite/python/autodoc_runme.py b/Examples/test-suite/python/autodoc_runme.py
index b0359f2e0..4c28f7f7e 100644
--- a/Examples/test-suite/python/autodoc_runme.py
+++ b/Examples/test-suite/python/autodoc_runme.py
@@ -1,14 +1,12 @@
 from autodoc import *
+import commentVerifier
 
-def check(got, expected):
-  if expected != got:
-    raise RuntimeError("\n" + "Expected: [" + str(expected) + "]\n" + "Got     : [" + str(got) + "]")
 
-check(A.__doc__, "Proxy of C++ A class")
-check(A.funk.__doc__, "just a string")
-check(A.func0.__doc__, "func0(self, arg2, hello) -> int")
-check(A.func1.__doc__, "func1(A self, short arg2, Tuple hello) -> int")
-check(A.func2.__doc__,  "\n"
+commentVerifier.check(A.__doc__, "Proxy of C++ A class")
+commentVerifier.check(A.funk.__doc__, "just a string")
+commentVerifier.check(A.func0.__doc__, "func0(self, arg2, hello) -> int")
+commentVerifier.check(A.func1.__doc__, "func1(A self, short arg2, Tuple hello) -> int")
+commentVerifier.check(A.func2.__doc__,  "\n"
 "        func2(self, arg2, hello) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -17,7 +15,7 @@ check(A.func2.__doc__,  "\n"
 "\n"
 "        "
 )
-check(A.func3.__doc__,  "\n"
+commentVerifier.check(A.func3.__doc__,  "\n"
 "        func3(A self, short arg2, Tuple hello) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -27,17 +25,17 @@ check(A.func3.__doc__,  "\n"
 "        "
 )
 
-check(A.func0default.__doc__, "\n"
+commentVerifier.check(A.func0default.__doc__, "\n"
 "        func0default(self, e, arg3, hello, f=2) -> int\n"
 "        func0default(self, e, arg3, hello) -> int\n"
 "        "
 )
-check(A.func1default.__doc__, "\n"
+commentVerifier.check(A.func1default.__doc__, "\n"
 "        func1default(A self, A e, short arg3, Tuple hello, double f=2) -> int\n"
 "        func1default(A self, A e, short arg3, Tuple hello) -> int\n"
 "        "
 )
-check(A.func2default.__doc__,  "\n"
+commentVerifier.check(A.func2default.__doc__,  "\n"
 "        func2default(self, e, arg3, hello, f=2) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -55,7 +53,7 @@ check(A.func2default.__doc__,  "\n"
 "\n"
 "        "
 )
-check(A.func3default.__doc__,  "\n"
+commentVerifier.check(A.func3default.__doc__,  "\n"
 "        func3default(A self, A e, short arg3, Tuple hello, double f=2) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -74,17 +72,17 @@ check(A.func3default.__doc__,  "\n"
 "        "
 )
 
-check(A.func0static.__doc__, "\n"
+commentVerifier.check(A.func0static.__doc__, "\n"
 "        func0static(e, arg2, hello, f=2) -> int\n"
 "        func0static(e, arg2, hello) -> int\n"
 "        "
 )
-check(A.func1static.__doc__, "\n"
+commentVerifier.check(A.func1static.__doc__, "\n"
 "        func1static(A e, short arg2, Tuple hello, double f=2) -> int\n"
 "        func1static(A e, short arg2, Tuple hello) -> int\n"
 "        "
 )
-check(A.func2static.__doc__,  "\n"
+commentVerifier.check(A.func2static.__doc__,  "\n"
 "        func2static(e, arg2, hello, f=2) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -102,7 +100,7 @@ check(A.func2static.__doc__,  "\n"
 "\n"
 "        "
 )
-check(A.func3static.__doc__,  "\n"
+commentVerifier.check(A.func3static.__doc__,  "\n"
 "        func3static(A e, short arg2, Tuple hello, double f=2) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -121,16 +119,16 @@ check(A.func3static.__doc__,  "\n"
 "        "
 )
 
-check(A.variable_a.__doc__, "A_variable_a_get(self) -> int")
-check(A.variable_b.__doc__, "A_variable_b_get(A self) -> int")
-check(A.variable_c.__doc__, "\n"
+commentVerifier.check(A.variable_a.__doc__, "A_variable_a_get(self) -> int")
+commentVerifier.check(A.variable_b.__doc__, "A_variable_b_get(A self) -> int")
+commentVerifier.check(A.variable_c.__doc__, "\n"
 "A_variable_c_get(self) -> int\n"
 "\n"
 "Parameters:\n"
 "    self: A *\n"
 "\n"
 )
-check(A.variable_d.__doc__, "\n"
+commentVerifier.check(A.variable_d.__doc__, "\n"
 "A_variable_d_get(A self) -> int\n"
 "\n"
 "Parameters:\n"
@@ -138,10 +136,10 @@ check(A.variable_d.__doc__, "\n"
 "\n"
 )
 
-check(B.__doc__, "Proxy of C++ B class")
-check(C.__init__.__doc__, "__init__(self, a, b, h) -> C")
-check(D.__init__.__doc__, "__init__(D self, int a, int b, Hola h) -> D")
-check(E.__init__.__doc__, "\n"
+commentVerifier.check(B.__doc__, "Proxy of C++ B class")
+commentVerifier.check(C.__init__.__doc__, "__init__(self, a, b, h) -> C")
+commentVerifier.check(D.__init__.__doc__, "__init__(D self, int a, int b, Hola h) -> D")
+commentVerifier.check(E.__init__.__doc__, "\n"
 "        __init__(self, a, b, h) -> E\n"
 "\n"
 "        Parameters:\n"
@@ -151,7 +149,7 @@ check(E.__init__.__doc__, "\n"
 "\n"
 "        "
 )
-check(F.__init__.__doc__, "\n"
+commentVerifier.check(F.__init__.__doc__, "\n"
 "        __init__(F self, int a, int b, Hola h) -> F\n"
 "\n"
 "        Parameters:\n"
@@ -162,15 +160,15 @@ check(F.__init__.__doc__, "\n"
 "        "
 )
 
-check(B.funk.__doc__, "funk(B self, int c, int d) -> int")
-check(funk.__doc__, "funk(A e, short arg2, int c, int d) -> int")
-check(funkdefaults.__doc__, "\n"
+commentVerifier.check(B.funk.__doc__, "funk(B self, int c, int d) -> int")
+commentVerifier.check(funk.__doc__, "funk(A e, short arg2, int c, int d) -> int")
+commentVerifier.check(funkdefaults.__doc__, "\n"
 "  funkdefaults(A e, short arg2, int c, int d, double f=2) -> int\n"
 "  funkdefaults(A e, short arg2, int c, int d) -> int\n"
 "  "
 )
 
-check(func_input.__doc__, "func_input(int * INPUT) -> int")
-check(func_output.__doc__, "func_output() -> int")
-check(func_inout.__doc__, "func_inout(int * INOUT) -> int")
-check(banana.__doc__, "banana(S a, S b, int c, Integer d)")
+commentVerifier.check(func_input.__doc__, "func_input(int * INPUT) -> int")
+commentVerifier.check(func_output.__doc__, "func_output() -> int")
+commentVerifier.check(func_inout.__doc__, "func_inout(int * INOUT) -> int")
+commentVerifier.check(banana.__doc__, "banana(S a, S b, int c, Integer d)")
diff --git a/Examples/test-suite/python/doxygen_misc_constructs_runme.py b/Examples/test-suite/python/doxygen_misc_constructs_runme.py
index 1c2c55236..61060e6be 100755
--- a/Examples/test-suite/python/doxygen_misc_constructs_runme.py
+++ b/Examples/test-suite/python/doxygen_misc_constructs_runme.py
@@ -110,6 +110,35 @@ commentVerifier.check(doxygen_misc_constructs.backslashC.__doc__,
   _with_ old comment parser. 
 
   See also: MyClass::fun(char,
-  float)
+   float)
   """
 )
+
+
+commentVerifier.check(doxygen_misc_constructs.cycle.__doc__,
+  r"""
+  The next line contains expression:
+
+  ['retVal < 10', 'g_counter == 23 && g_mode & 3']
+
+
+  Both words should be emphasized __isystem.connect__.
+  But not the last period. For __example__, comma should not be emphasized.
+  Similar __for__: double colon.
+
+  Spaces at the start of line should be taken into account:
+  Arguments:
+    id (int) -- used as prefix in log
+      statements. The default value is empty string, which is OK if
+      there is only one app. instance. Example:
+      
+          ctrl.setBP("func1");
+      
+      If we set the id to 'main_', we get:
+      
+          main_ctrl.setBP("func1");
+      
+
+    fileName (char *) -- name of the log file
+  """
+);
diff --git a/Examples/test-suite/python/doxygen_translate_all_tags_runme.py b/Examples/test-suite/python/doxygen_translate_all_tags_runme.py
index 3009caf64..051a9a51e 100755
--- a/Examples/test-suite/python/doxygen_translate_all_tags_runme.py
+++ b/Examples/test-suite/python/doxygen_translate_all_tags_runme.py
@@ -67,7 +67,7 @@ r"""
 
 commentVerifier.check(doxygen_translate_all_tags.func03.__doc__,
 r"""
-  Comment for __func03().__
+  Comment for __func03()__.
 
 
 
@@ -96,11 +96,11 @@ r"""
     \sqrt{(x_2-x_1)^2+(y_2-y_1)^2} 
 
    
-  \sqrt{(x_2-x_1)^2+(y_2-y_1)^2}
+      \sqrt{(x_2-x_1)^2+(y_2-y_1)^2}
 
 
    
-  \sqrt{(x_2-x_1)^2+(y_2-y_1)^2}
+      \sqrt{(x_2-x_1)^2+(y_2-y_1)^2}
 
 
 
@@ -120,21 +120,21 @@ r"""
 commentVerifier.check(doxygen_translate_all_tags.func05.__doc__,
 r"""
   If: ANOTHERCONDITION {
-  First part of comment
-  If: SECONDCONDITION {
-  Nested condition text
-  }Else if: THIRDCONDITION {
-  The third condition text
-  }Else:  {The last text block
-  }
-  }Else:  {Second part of comment
-  If: CONDITION {
-  Second part extended
-  }
+    First part of comment
+    If: SECONDCONDITION {
+      Nested condition text
+    }Else if: THIRDCONDITION {
+      The third condition text
+    }Else:  {    The last text block
+    }
+  }Else:  {  Second part of comment
+    If: CONDITION {
+      Second part extended
+    }
   }
 
   If not: SOMECONDITION {
-  This is printed if not
+    This is printed if not
   }
 
   Image: testImage.bmp("Hello, world!")
@@ -155,7 +155,7 @@ r"""
 
 commentVerifier.check(doxygen_translate_all_tags.func06.__doc__,
 r"""
-  Comment for __func06().__
+  Comment for __func06()__.
 
 
 
@@ -194,7 +194,7 @@ r"""
 
 commentVerifier.check(doxygen_translate_all_tags.func07.__doc__,
 r"""
-  Comment for __func07().__
+  Comment for __func07()__.
 
 
 
diff --git a/Examples/test-suite/python/doxygen_translate_runme.py b/Examples/test-suite/python/doxygen_translate_runme.py
index 183558415..9365d1ce6 100755
--- a/Examples/test-suite/python/doxygen_translate_runme.py
+++ b/Examples/test-suite/python/doxygen_translate_runme.py
@@ -40,21 +40,21 @@ r"""
   Throws: SuperError
 
   If: ANOTHERCONDITION {
-  First part of comment
-  If: SECONDCONDITION {
-  Nested condition text
-  }Else if: THIRDCONDITION {
-  The third condition text
-  }Else:  {The last text block
-  }
-  }Else:  {Second part of comment
-  If: CONDITION {
-  Second part extended
-  }
+    First part of comment
+    If: SECONDCONDITION {
+      Nested condition text
+    }Else if: THIRDCONDITION {
+      The third condition text
+    }Else:  {    The last text block
+    }
+  }Else:  {  Second part of comment
+    If: CONDITION {
+      Second part extended
+    }
   }
 
   If not: SOMECONDITION {
-  This is printed if not
+    This is printed if not
   }
 
   Image: testImage.bmp("Hello, world!")
diff --git a/Source/DoxygenTranslator/src/DoxygenParser.cpp b/Source/DoxygenTranslator/src/DoxygenParser.cpp
index d270631df..07839f5d7 100644
--- a/Source/DoxygenTranslator/src/DoxygenParser.cpp
+++ b/Source/DoxygenTranslator/src/DoxygenParser.cpp
@@ -199,39 +199,33 @@ std::string DoxygenParser::getNextWord() {
         return "";
     }
 */
-	while (m_tokenListIt != m_tokenList.end()  &&  (m_tokenListIt->m_tokenType == PLAINSTRING)) {
-		// handle quoted strings as words
-	    string token = m_tokenListIt->m_tokenString;
-		if (token == "\"") {
-
-			string word = m_tokenListIt->m_tokenString;
-			m_tokenListIt++;
-			while (true) {
-				string nextWord = getNextToken();
-				if (nextWord.empty()) { // maybe report unterminated string error
-					return word;
-				}
-				word += nextWord;
-				if (nextWord == "\"") {
-					return word;
-				}
-			}
-		}
+  while (m_tokenListIt != m_tokenList.end()  &&  (m_tokenListIt->m_tokenType == PLAINSTRING)) {
+    // handle quoted strings as words
+    string token = m_tokenListIt->m_tokenString;
+    if (token == "\"") {
+
+      string word = m_tokenListIt->m_tokenString;
+      m_tokenListIt++;
+      while (true) {
+        string nextWord = getNextToken();
+        if (nextWord.empty()) { // maybe report unterminated string error
+          return word;
+        }
+        word += nextWord;
+        if (nextWord == "\"") {
+          return word;
+        }
+      }
+    }
 
-        string tokenStr = trim(m_tokenListIt->m_tokenString);
-		m_tokenListIt++;
-		if (!tokenStr.empty()) {
-		    return tokenStr;
-		}
-	} /* else if (nextToken.m_tokenType == END_LINE) {
-	    // this handles cases when command is the last item in line, for example:
-	    // * This method returns line number \c
-	    // * relative to paragraph.
-        m_tokenListIt++;
-        return getNextWord();
-	} */
+    string tokenStr = trim(m_tokenListIt->m_tokenString);
+    m_tokenListIt++;
+    if (!tokenStr.empty()) {
+      return tokenStr;
+    }
+  }
 
-	return "";
+  return "";
 }
 
 
@@ -381,36 +375,36 @@ void DoxygenParser::skipEndOfLine()
 
 int DoxygenParser::addSimpleCommand(const std::string &theCommand,
                                          DoxygenEntityList &doxyList) {
-	if (noisy)
-		cout << "Parsing " << theCommand << endl;
+  if (noisy)
+    cout << "Parsing " << theCommand << endl;
 
-	doxyList.push_back(DoxygenEntity(theCommand));
-	return 1;
+  doxyList.push_back(DoxygenEntity(theCommand));
+  return 1;
 }
 
 
 int DoxygenParser::addCommandWord(const std::string &theCommand,
                                        const TokenList &,
                                        DoxygenEntityList &doxyList) {
-	if (noisy)
-		cout << "Parsing " << theCommand << endl;
+  if (noisy)
+    cout << "Parsing " << theCommand << endl;
 
-	if (isEndOfLine()) {
-	    // handles cases when command is at the end of line (for example "\c\nreally"
-	    skipWhitespaceTokens();
-	    doxyList.push_back(DoxygenEntity("plainstd::endl"));
-	}
-	std::string name = getNextWord();
-    if (!name.empty()) {
-      DoxygenEntityList aNewList;
-      aNewList.push_back(DoxygenEntity("plainstd::string", name));
-      doxyList.push_back(DoxygenEntity(theCommand, aNewList));
-      return 1;
-	} else {
-		printListError(WARN_DOXYGEN_COMMAND_ERROR, "No word followed " +
-		                theCommand + " command. Not added");
-	}
-	return 0;
+  if (isEndOfLine()) {
+    // handles cases when command is at the end of line (for example "\c\nreally"
+    skipWhitespaceTokens();
+    doxyList.push_back(DoxygenEntity("plainstd::endl"));
+  }
+  std::string name = getNextWord();
+  if (!name.empty()) {
+    DoxygenEntityList aNewList;
+    aNewList.push_back(DoxygenEntity("plainstd::string", name));
+    doxyList.push_back(DoxygenEntity(theCommand, aNewList));
+    return 1;
+  } else {
+    printListError(WARN_DOXYGEN_COMMAND_ERROR, "No word followed " +
+        theCommand + " command. Not added");
+  }
+  return 0;
 }
 
 
@@ -1264,32 +1258,69 @@ void DoxygenParser::tokenizeDoxygenComment(const std::string &doxygenComment,
     const string &line = *it;
     size_t pos = line.find_first_not_of(" \t");
 
+    if (pos == string::npos) {
+      m_tokenList.push_back(Token(END_LINE, "\n"));
+      continue;
+    }
+
     // skip sequences of '*', '/', and '!' of any length
-    while (pos != string::npos  &&  isStartOfDoxyCommentChar(line[pos])) {
+    bool isStartOfCommentLineCharFound = false;
+    while (pos < line.size()  &&  isStartOfDoxyCommentChar(line[pos])) {
       pos++;
+      isStartOfCommentLineCharFound = true;
     }
 
-    if (pos == string::npos) {
-        m_tokenList.push_back(Token(END_LINE, "\n"));
+    if (pos == line.size()) {
+      m_tokenList.push_back(Token(END_LINE, "\n"));
       continue;
     }
 
+    // if 'isStartOfCommentLineCharFound' then preserve leading spaces, so
+    // ' *    comment' gets translated to ' *    comment', not ' * comment'
+    // This is important to keep formatting for comments translated to Python.
+    if (isStartOfCommentLineCharFound  &&  line[pos] == ' ') {
+      pos++; // points to char after ' * '
+      if (pos == line.size()) {
+        m_tokenList.push_back(Token(END_LINE, "\n"));
+        continue;
+      }
+    }
+
     // line[pos] may be ' \t' or start of word, it there was no '*', '/' or '!'
     // at beginning of the line. Make sure it points to start of the first word
     // in the line.
-    pos = line.find_first_not_of(" \t", pos);
-    if (pos == string::npos) {
+    size_t firstWordPos = line.find_first_not_of(" \t", pos);
+    if (firstWordPos == string::npos) {
       m_tokenList.push_back(Token(END_LINE, "\n"));
       continue;
     }
 
+    if (isStartOfCommentLineCharFound  &&  firstWordPos > pos) {
+      m_tokenList.push_back(Token(PLAINSTRING, line.substr(pos, firstWordPos - pos)));
+    }
+
+    pos = firstWordPos;
+
     while (pos != string::npos) {
       // find the end of the word
       size_t doxyCmdOrHtmlTagPos = line.find_first_of("\\@<>&\" \t", pos);
       if (doxyCmdOrHtmlTagPos != pos) {
         // plain text found
-        m_tokenList.push_back(Token(PLAINSTRING,
-                              line.substr(pos, doxyCmdOrHtmlTagPos - pos)));
+        // if the last char is punctuation, make it a separate word, otherwise
+        // it may be included with word also when not appropriate, for example:
+        //   colors are \b red, green, and blue --> colors are <b>red,</b> green, and blue
+        // instead of (comma not bold):
+        //   colors are \b red, green, and blue --> colors are <b>red</b>, green, and blue
+        // In Python it looks even worse:
+        //   colors are \b red, green, and blue --> colors are 'red,' green, and blue
+        string text = line.substr(pos, doxyCmdOrHtmlTagPos - pos);
+        string punctuations(".,:");
+        if (!text.empty()  &&  punctuations.find(text[text.size() - 1]) != string::npos) {
+          m_tokenList.push_back(Token(PLAINSTRING, text.substr(0, text.size() - 1)));
+          m_tokenList.push_back(Token(PLAINSTRING, text.substr(text.size() - 1)));
+        } else {
+          m_tokenList.push_back(Token(PLAINSTRING, text));
+        }
       }
 
       pos = doxyCmdOrHtmlTagPos;