Merge pull request #1310 from pygments/815/MartijnBraam/dmesg

This commit adds a lexer for linux kernel logs as outputted by `dmesg`

10 files changed, 115 insertions, 9 deletions

diff --git a/AUTHORS b/AUTHORS
index 751c9172..58968f50 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -35,6 +35,7 @@ Other contributors, listed alphabetically, are:
 * Stéphane Blondon -- SGF lexer
 * Frits van Bommel -- assembler lexers
 * Pierre Bourdon -- bugfixes
+* Martijn Braam -- Kernel log lexer
 * Matthias Bussonnier -- ANSI style handling for terminal-256 formatter
 * chebee7i -- Python traceback lexer improvements
 * Hiram Chirino -- Scaml and Jade lexers
diff --git a/CHANGES b/CHANGES
index 9337b5f0..6ac44302 100644
--- a/CHANGES
+++ b/CHANGES
@@ -13,6 +13,7 @@ Version 2.6
 
 - Added lexers:
 
+  * Linux kernel logs (PR#1310)
   * Ride (PR#1319, PR#1321)
   * USD (PR#1290)
   * Mosel (PR#1287, PR#1326)
diff --git a/doc/languages.rst b/doc/languages.rst
index f9492525..57bea8ea 100644
--- a/doc/languages.rst
+++ b/doc/languages.rst
@@ -235,6 +235,7 @@ Other markup
 * JSON, JSON-LD
 * Lean theorem prover
 * Lighttpd config files
+* Linux kernel log (dmesg)
 * LLVM assembly
 * LSL scripts
 * Makefiles
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 1ed7140d..9f306122 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -233,6 +233,7 @@ LEXERS = {
     'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle', 'juttle'), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')),
     'KalLexer': ('pygments.lexers.javascript', 'Kal', ('kal',), ('*.kal',), ('text/kal', 'application/kal')),
     'KconfigLexer': ('pygments.lexers.configs', 'Kconfig', ('kconfig', 'menuconfig', 'linux-config', 'kernel-config'), ('Kconfig', '*Config.in*', 'external.in*', 'standard-modules.in'), ('text/x-kconfig',)),
+    'KernelLogLexer': ('pygments.lexers.textfmts', 'Kernel log', ('kmsg', 'dmesg'), ('*.kmsg', '*.dmesg'), ()),
     'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)),
     'KotlinLexer': ('pygments.lexers.jvm', 'Kotlin', ('kotlin',), ('*.kt',), ('text/x-kotlin',)),
     'LSLLexer': ('pygments.lexers.scripting', 'LSL', ('lsl',), ('*.lsl',), ('text/x-lsl',)),
diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py
index e9cf6722..0e418535 100644
--- a/pygments/lexers/javascript.py
+++ b/pygments/lexers/javascript.py
@@ -259,11 +259,11 @@ class LiveScriptLexer(RegexLexer):
             (r'//', String.Regex, ('#pop', 'multilineregex')),
             (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
              r'([gim]+\b|\B)', String.Regex, '#pop'),
+            (r'/', Operator, '#pop'),
             default('#pop'),
         ],
         'root': [
-            # this next expr leads to infinite loops root -> slashstartsregex
-            # (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+            (r'^(?=\s|/)', Text, 'slashstartsregex'),
             include('commentsandwhitespace'),
             (r'(?:\([^()]+\))?[ ]*[~-]{1,2}>|'
              r'(?:\(?[^()\n]+\)?)?[ ]*<[~-]{1,2}', Name.Function),
@@ -1060,7 +1060,7 @@ class CoffeeScriptLexer(RegexLexer):
             # This isn't really guarding against mishighlighting well-formed
             # code, just the ability to infinite-loop between root and
             # slashstartsregex.
-            (r'/', Operator),
+            (r'/', Operator, '#pop'),
             default('#pop'),
         ],
         'root': [
diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py
index bd4a2376..680f0573 100644
--- a/pygments/lexers/php.py
+++ b/pygments/lexers/php.py
@@ -50,13 +50,14 @@ class ZephirLexer(RegexLexer):
             include('commentsandwhitespace'),
             (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
              r'([gim]+\b|\B)', String.Regex, '#pop'),
+            (r'/', Operator, '#pop'),
             default('#pop')
         ],
         'badregex': [
             (r'\n', Text, '#pop')
         ],
         'root': [
-            (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+            (r'^(?=\s|/)', Text, 'slashstartsregex'),
             include('commentsandwhitespace'),
             (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
              r'(<<|>>>?|==?|!=?|->|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
diff --git a/pygments/lexers/textfmts.py b/pygments/lexers/textfmts.py
index a310f8b5..bfd053d5 100644
--- a/pygments/lexers/textfmts.py
+++ b/pygments/lexers/textfmts.py
@@ -12,13 +12,13 @@
 import re
 
 from pygments.lexers import guess_lexer, get_lexer_by_name
-from pygments.lexer import RegexLexer, bygroups, default, do_insertions
+from pygments.lexer import RegexLexer, bygroups, default, include
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Generic, Literal, Punctuation
 from pygments.util import ClassNotFound
 
 __all__ = ['IrcLogsLexer', 'TodotxtLexer', 'HttpLexer', 'GettextLexer',
-           'NotmuchLexer']
+           'NotmuchLexer', 'KernelLogLexer']
 
 
 class IrcLogsLexer(RegexLexer):
@@ -380,3 +380,52 @@ class NotmuchLexer(RegexLexer):
     def __init__(self, **options):
         self.body_lexer = options.get('body_lexer', None)
         RegexLexer.__init__(self, **options)
+
+
+class KernelLogLexer(RegexLexer):
+    """
+    For Linux Kernel log ("dmesg") output.
+
+    .. versionadded:: 2.6
+    """
+    name = 'Kernel log'
+    aliases = ['kmsg', 'dmesg']
+    filenames = ['*.kmsg', '*.dmesg']
+
+    tokens = {
+        'root': [
+            (r'^[^:]+:debug : (?=\[)', Text, 'debug'),
+            (r'^[^:]+:info  : (?=\[)', Text, 'info'),
+            (r'^[^:]+:warn  : (?=\[)', Text, 'warn'),
+            (r'^[^:]+:notice: (?=\[)', Text, 'warn'),
+            (r'^[^:]+:err   : (?=\[)', Text, 'error'),
+            (r'^[^:]+:crit  : (?=\[)', Text, 'error'),
+            (r'^(?=\[)', Text, 'unknown'),
+        ],
+        'unknown': [
+            (r'^(?=.+(warning|notice|audit|deprecated))', Text, 'warn'),
+            (r'^(?=.+(error|critical|fail|Bug))', Text, 'error'),
+            default('info'),
+        ],
+        'base': [
+            (r'\[[0-9\. ]+\] ', Number),
+            (r'(?<=\] ).+?:', Keyword),
+            (r'\n', Text, '#pop'),
+        ],
+        'debug': [
+            include('base'),
+            (r'.+\n', Comment, '#pop')
+        ],
+        'info': [
+            include('base'),
+            (r'.+\n', Text, '#pop')
+        ],
+        'warn': [
+            include('base'),
+            (r'.+\n', Generic.Strong, '#pop')
+        ],
+        'error': [
+            include('base'),
+            (r'.+\n', Generic.Error, '#pop')
+        ]
+    }
diff --git a/pygments/lexers/webmisc.py b/pygments/lexers/webmisc.py
index b39334bc..448aff50 100644
--- a/pygments/lexers/webmisc.py
+++ b/pygments/lexers/webmisc.py
@@ -158,6 +158,9 @@ class XQueryLexer(ExtendedRegexLexer):
         # state stack
         if len(lexer.xquery_parse_state) == 0:
             ctx.stack.pop()
+            if not ctx.stack:
+                # make sure we have at least the root state on invalid inputs
+                ctx.stack = ['root']
         elif len(ctx.stack) > 1:
             ctx.stack.append(lexer.xquery_parse_state.pop())
         else:
diff --git a/tests/examplefiles/example.dmesg b/tests/examplefiles/example.dmesg
new file mode 100644
index 00000000..f023ff20
--- /dev/null
+++ b/tests/examplefiles/example.dmesg
@@ -0,0 +1,52 @@
+[    0.000000] Linux version 5.0.9-arch1-1-ARCH (builduser@heftig-18307) (gcc version 8.3.0 (GCC)) #1 SMP PREEMPT Sat Apr 20 15:00:46 UTC 2019
+[    0.000000] Command line: initrd=\initramfs-linux.img root=/dev/nvme0n1p1 nouveau.noaccel=1 rw
+[    0.000000] KERNEL supported cpus:
+[    0.000000]   Intel GenuineIntel
+[    0.000000]   AMD AuthenticAMD
+[    0.000000]   Hygon HygonGenuine
+[    0.000000]   Centaur CentaurHauls
+[    0.000000] x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers'
+[    0.000000] x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers'
+[    0.000000] x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers'
+[    0.000000] x86/fpu: Supporting XSAVE feature 0x008: 'MPX bounds registers'
+[    0.000000] x86/fpu: Supporting XSAVE feature 0x010: 'MPX CSR'
+[    0.000000] x86/fpu: xstate_offset[2]:  576, xstate_sizes[2]:  256
+[    0.000000] x86/fpu: xstate_offset[3]:  832, xstate_sizes[3]:   64
+[    0.000000] x86/fpu: xstate_offset[4]:  896, xstate_sizes[4]:   64
+[    0.000000] x86/fpu: Enabled xstate features 0x1f, context size is 960 bytes, using 'compacted' format.
+[    2.663456] tpm_crb MSFT0101:00: [Firmware Bug]: ACPI region does not cover the entire command/response buffer. [mem 0xfed40000-0xfed4087f flags 0x200] vs fed40080 f80
+[    2.663514] tpm_crb MSFT0101:00: [Firmware Bug]: ACPI region does not cover the entire command/response buffer. [mem 0xfed40000-0xfed4087f flags 0x200] vs fed40080 f80
+[    2.664809] Bluetooth: Core ver 2.22
+[    2.664820] NET: Registered protocol family 31
+[ 3134.452501] usb 2-2: new SuperSpeed Gen 1 USB device number 2 using xhci_hcd
+[ 3134.471506] usb 2-2: New USB device found, idVendor=0781, idProduct=cfd2, bcdDevice= 0.02
+[ 3134.471508] usb 2-2: New USB device strings: Mfr=3, Product=4, SerialNumber=2
+
+kern  :notice: [    0.000000] Linux version 5.0.9-arch1-1-ARCH (builduser@heftig-18307) (gcc version 8.3.0 (GCC)) #1 SMP PREEMPT Sat Apr 20 15:00:46 UTC 2019
+kern  :info  : [    0.000000] Command line: initrd=\initramfs-linux.img root=/dev/nvme0n1p1 nouveau.noaccel=1 rw
+kern  :info  : [    0.000000] KERNEL supported cpus:
+kern  :info  : [    0.000000]   Intel GenuineIntel
+kern  :info  : [    0.000000]   AMD AuthenticAMD
+kern  :info  : [    0.000000]   Hygon HygonGenuine
+kern  :info  : [    0.000000]   Centaur CentaurHauls
+kern  :info  : [    0.000000] x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers'
+kern  :info  : [    0.000000] x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers'
+kern  :info  : [    0.000000] x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers'
+kern  :info  : [    0.000000] x86/fpu: Supporting XSAVE feature 0x008: 'MPX bounds registers'
+kern  :info  : [    0.000000] x86/fpu: Supporting XSAVE feature 0x010: 'MPX CSR'
+kern  :info  : [    0.000000] x86/fpu: xstate_offset[2]:  576, xstate_sizes[2]:  256
+kern  :info  : [    0.000000] x86/fpu: xstate_offset[3]:  832, xstate_sizes[3]:   64
+kern  :info  : [    0.000000] x86/fpu: xstate_offset[4]:  896, xstate_sizes[4]:   64
+kern  :info  : [    0.000000] x86/fpu: Enabled xstate features 0x1f, context size is 960 bytes, using 'compacted' format.
+kern  :info  : [    0.000000] BIOS-provided physical RAM map:
+kern  :info  : [    0.000000] BIOS-e820: [mem 0x0000000000000000-0x0000000000057fff] usable
+kern  :info  : [    2.382262] EXT4-fs (nvme0n1p1): re-mounted. Opts: (null)
+kern  :notice: [    2.389774] random: systemd-random-: uninitialized urandom read (512 bytes read)
+kern  :info  : [    2.397148] usb 1-10: New USB device found, idVendor=8087, idProduct=0aa7, bcdDevice= 0.01
+kern  :info  : [    2.397150] usb 1-10: New USB device strings: Mfr=0, Product=0, SerialNumber=0
+kern  :info  : [    2.519599] usb 1-12: new high-speed USB device number 4 using xhci_hcd
+kern  :crit  : [18706.135478] mce: CPU6: Package temperature above threshold, cpu clock throttled (total events = 79)
+kern  :crit  : [18706.135479] mce: CPU3: Package temperature above threshold, cpu clock throttled (total events = 79)
+kern  :crit  : [18706.135484] mce: CPU4: Package temperature above threshold, cpu clock throttled (total events = 79)
+kern  :info  : [18706.136450] mce: CPU4: Core temperature/speed normal
+kern  :info  : [18706.136451] mce: CPU1: Package temperature/speed normal
diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py
index 99d56bb5..7d08c05e 100644
--- a/tests/test_basic_api.py
+++ b/tests/test_basic_api.py
@@ -64,9 +64,6 @@ def test_lexer_classes(cls):
 
 @pytest.mark.parametrize('cls', lexers._iter_lexerclasses(plugins=False))
 def test_random_input(cls):
-    if cls.name in ['XQuery', 'Opa', 'Zephir']:   # XXX temporary
-        return
-
     inst = cls()
     try:
         tokens = list(inst.get_tokens(test_content))