diff options
Diffstat (limited to 'fail2ban/client')
| -rw-r--r-- | fail2ban/client/actionreader.py | 21 | ||||
| -rw-r--r-- | fail2ban/client/beautifier.py | 6 | ||||
| -rw-r--r-- | fail2ban/client/configparserinc.py | 19 | ||||
| -rw-r--r-- | fail2ban/client/configreader.py | 82 | ||||
| -rw-r--r-- | fail2ban/client/csocket.py | 14 | ||||
| -rwxr-xr-x | fail2ban/client/fail2banclient.py | 75 | ||||
| -rw-r--r-- | fail2ban/client/fail2bancmdline.py | 60 | ||||
| -rw-r--r-- | fail2ban/client/fail2banreader.py | 16 | ||||
| -rw-r--r-- | fail2ban/client/fail2banregex.py | 412 | ||||
| -rw-r--r-- | fail2ban/client/fail2banserver.py | 28 | ||||
| -rw-r--r-- | fail2ban/client/filterreader.py | 38 | ||||
| -rw-r--r-- | fail2ban/client/jailreader.py | 130 |
12 files changed, 582 insertions, 319 deletions
diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 3ed8204c..88b0aca1 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -38,26 +38,32 @@ class ActionReader(DefinitionInitConfigReader): _configOpts = { "actionstart": ["string", None], - "actionstart_on_demand": ["string", None], + "actionstart_on_demand": ["bool", None], "actionstop": ["string", None], "actionflush": ["string", None], "actionreload": ["string", None], "actioncheck": ["string", None], "actionrepair": ["string", None], + "actionrepair_on_unban": ["bool", None], "actionban": ["string", None], "actionprolong": ["string", None], + "actionreban": ["string", None], "actionunban": ["string", None], - "norestored": ["string", None], + "norestored": ["bool", None], } def __init__(self, file_, jailName, initOpts, **kwargs): + # always supply jail name as name parameter if not specified in options: + n = initOpts.get("name") + if n is None: + initOpts["name"] = n = jailName actname = initOpts.get("actname") if actname is None: actname = file_ + # ensure we've unique action name per jail: + if n != jailName: + actname += n[len(jailName):] if n.startswith(jailName) else '-' + n initOpts["actname"] = actname - # always supply jail name as name parameter if not specified in options: - if initOpts.get("name") is None: - initOpts["name"] = jailName self._name = actname DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) @@ -78,11 +84,6 @@ class ActionReader(DefinitionInitConfigReader): def convert(self): opts = self.getCombined( ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) - # type-convert only after combined (otherwise boolean converting prevents substitution): - for o in ('norestored', 'actionstart_on_demand'): - if opts.get(o): - opts[o] = self._convert_to_boolean(opts[o]) - # stream-convert: head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 4d9e549f..97cd38b2 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -180,6 +180,12 @@ class Beautifier: msg = "The jail %s action %s has the following " \ "methods:\n" % (inC[1], inC[3]) msg += ", ".join(response) + elif inC[2] == "banip" and inC[0] == "get": + if isinstance(response, list): + sep = " " if len(inC) <= 3 else inC[3] + if sep == "--with-time": + sep = "\n" + msg = sep.join(response) except Exception: logSys.warning("Beautifier error. Please report the error") logSys.error("Beautify %r with %r failed", response, self.__inputCmd, diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 70dfd91b..cc4ada0a 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -29,7 +29,7 @@ import re import sys from ..helpers import getLogger -if sys.version_info >= (3,2): +if sys.version_info >= (3,): # pragma: 2.x no cover # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ @@ -61,7 +61,7 @@ if sys.version_info >= (3,2): return super(BasicInterpolationWithName, self)._interpolate_some( parser, option, accum, rest, section, map, *args, **kwargs) -else: # pragma: no cover +else: # pragma: 3.x no cover from ConfigParser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError @@ -73,6 +73,17 @@ else: # pragma: no cover return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs) SafeConfigParser._interpolate_some = _interpolate_some +def _expandConfFilesWithLocal(filenames): + """Expands config files with local extension. + """ + newFilenames = [] + for filename in filenames: + newFilenames.append(filename) + localname = os.path.splitext(filename)[0] + '.local' + if localname not in filenames and os.path.isfile(localname): + newFilenames.append(localname) + return newFilenames + # Gets the instance of the logger. logSys = getLogger(__name__) logLevel = 7 @@ -245,6 +256,7 @@ after = 1.conf def _getIncludes(self, filenames, seen=[]): if not isinstance(filenames, list): filenames = [ filenames ] + filenames = _expandConfFilesWithLocal(filenames) # retrieve or cache include paths: if self._cfg_share: # cache/share include list: @@ -360,7 +372,8 @@ after = 1.conf s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: - self.merge_section('KNOWN/'+n, s2, '') + self.merge_section('KNOWN/'+n, + dict(filter(lambda i: i[0] in s, s2.iteritems())), '') # merge section s2.update(s) else: diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index b03daca9..c7f965ce 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) +CONVERTER = { + "bool": _as_bool, + "int": int, +} +def _OptionsTemplateGen(options): + """Iterator over the options template with default options. + + Each options entry is composed of an array or tuple with: + [[type, name, ?default?], ...] + Or it is a dict: + {name: [type, default], ...} + """ + if isinstance(options, (list,tuple)): + for optname in options: + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + yield opttype, optname, optvalue + else: + for optname in options: + opttype, optvalue = options[optname] + yield opttype, optname, optvalue + class ConfigReader(): """Generic config reader class. @@ -120,6 +144,13 @@ class ConfigReader(): except AttributeError: return False + def has_option(self, sec, opt, withDefault=True): + return self._cfg.has_option(sec, opt) if withDefault \ + else opt in self._cfg._sections.get(sec, {}) + + def merge_defaults(self, d): + self._cfg.get_defaults().update(d) + def merge_section(self, section, *args, **kwargs): try: return self._cfg.merge_section(section, *args, **kwargs) @@ -221,29 +252,22 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None, shouldExist=False): + def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True): values = dict() if pOptions is None: pOptions = {} # Get only specified options: - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] + for opttype, optname, optvalue in _OptionsTemplateGen(options): if optname in pOptions: continue try: - if opttype == "bool": - v = self.getboolean(sec, optname) - elif opttype == "int": - v = self.getint(sec, optname) - else: - v = self.get(sec, optname, vars=pOptions) + v = self.get(sec, optname, vars=pOptions) values[optname] = v + if convert: + conv = CONVERTER.get(opttype) + if conv: + if v is None: continue + values[optname] = conv(v) except NoSectionError as e: if shouldExist: raise @@ -253,11 +277,11 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # TODO: validate error handling here. except NoOptionError: if not optvalue is None: - logSys.warning("'%s' not defined in '%s'. Using default one: %r" + logSys.debug("'%s' not defined in '%s'. Using default one: %r" % (optname, sec, optvalue)) values[optname] = optvalue - elif logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) + # elif logSys.getEffectiveLevel() <= logLevel: + # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") @@ -315,8 +339,9 @@ class DefinitionInitConfigReader(ConfigReader): pOpts = dict() if self._initOpts: pOpts = _merge_dicts(pOpts, self._initOpts) + # type-convert only in combined (otherwise int/bool converting prevents substitution): self._opts = ConfigReader.getOptions( - self, "Definition", self._configOpts, pOpts) + self, "Definition", self._configOpts, pOpts, convert=False) self._pOpts = pOpts if self.has_section("Init"): # get only own options (without options from default): @@ -337,10 +362,21 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) + def convertOptions(self, opts, configOpts): + """Convert interpolated combined options to expected type. + """ + for opttype, optname, optvalue in _OptionsTemplateGen(configOpts): + conv = CONVERTER.get(opttype) + if conv: + v = opts.get(optname) + if v is None: continue + try: + opts[optname] = conv(v) + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue - def _convert_to_boolean(self, value): - return _as_bool(value) - def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init options as preselection (values with higher precedence as specified in section). @@ -375,6 +411,8 @@ class DefinitionInitConfigReader(ConfigReader): ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') + # convert options after all interpolations: + self.convertOptions(opts, self._configOpts) return opts def convert(self): diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index ab3e294b..88795674 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -48,7 +48,8 @@ class CSocket: def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) - self.__csock.send(obj + CSPROTO.END) + self.__csock.send(obj) + self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) def settimeout(self, timeout): @@ -81,9 +82,12 @@ class CSocket: msg = CSPROTO.EMPTY if nonblocking: sock.setblocking(0) if timeout: sock.settimeout(timeout) - while msg.rfind(CSPROTO.END) == -1: - chunk = sock.recv(512) - if chunk in ('', b''): # python 3.x may return b'' instead of '' - raise RuntimeError("socket connection broken") + bufsize = 1024 + while msg.rfind(CSPROTO.END, -32) == -1: + chunk = sock.recv(bufsize) + if not len(chunk): + raise socket.error(104, 'Connection reset by peer') + if chunk == CSPROTO.END: break msg = msg + chunk + if bufsize < 32768: bufsize <<= 1 return loads(msg) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 73abacb2..f3b0f7b2 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -168,30 +168,20 @@ class Fail2banClient(Fail2banCmdLine, Thread): if not ret: return None - # verify that directory for the socket file exists - socket_dir = os.path.dirname(self._conf["socket"]) - if not os.path.exists(socket_dir): - logSys.error( - "There is no directory %s to contain the socket file %s." - % (socket_dir, self._conf["socket"])) - return None - if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover - logSys.error( - "Directory %s exists but not accessible for writing" - % (socket_dir,)) - return None - # Check already running if not self._conf["force"] and os.path.exists(self._conf["socket"]): logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)") return None - stream.append(['server-status']) - return stream + return [["server-stream", stream], ['server-status']] + + def _set_server(self, s): + self._server = s ## def __startServer(self, background=True): from .fail2banserver import Fail2banServer + # read configuration here (in client only, in server we do that in the config-thread): stream = self.__prepareStartServer() self._alive = True if not stream: @@ -206,16 +196,19 @@ class Fail2banClient(Fail2banCmdLine, Thread): return False else: # In foreground mode we should make server/client communication in different threads: - th = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)) - th.daemon = True - th.start() + phase = dict() + self.configureServer(phase=phase, stream=stream) # Mark current (main) thread as daemon: - self.setDaemon(True) + self.daemon = True # Start server direct here in main thread (not fork): - self._server = Fail2banServer.startServerDirect(self._conf, False) - + self._server = Fail2banServer.startServerDirect(self._conf, False, self._set_server) + if not phase.get('done', False): + if self._server: # pragma: no cover + self._server.quit() + self._server = None + exit(255) except ExitException: # pragma: no cover - pass + raise except Exception as e: # pragma: no cover output("") logSys.error("Exception while starting server " + ("background" if background else "foreground")) @@ -228,23 +221,39 @@ class Fail2banClient(Fail2banCmdLine, Thread): return True ## - def configureServer(self, nonsync=True, phase=None): + def configureServer(self, nonsync=True, phase=None, stream=None): # if asynchronous start this operation in the new thread: if nonsync: - th = Thread(target=Fail2banClient.configureServer, args=(self, False, phase)) + if phase is not None: + # event for server ready flag: + def _server_ready(): + phase['start-ready'] = True + logSys.log(5, ' server phase %s', phase) + # notify waiting thread if server really ready + self._conf['onstart'] = _server_ready + th = Thread(target=Fail2banClient.configureServer, args=(self, False, phase, stream)) th.daemon = True - return th.start() + th.start() + # if we need to read configuration stream: + if stream is None and phase is not None: + # wait, do not continue if configuration is not 100% valid: + Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"], 0.001) + logSys.log(5, ' server phase %s', phase) + if not phase.get('start', False): + raise ServerExecutionException('Async configuration of server failed') + return True # prepare: read config, check configuration is valid, etc.: if phase is not None: phase['start'] = True logSys.log(5, ' client phase %s', phase) - stream = self.__prepareStartServer() + if stream is None: + stream = self.__prepareStartServer() if phase is not None: phase['ready'] = phase['start'] = (True if stream else False) logSys.log(5, ' client phase %s', phase) if not stream: return False - # wait a litle bit for phase "start-ready" before enter active waiting: + # wait a little bit for phase "start-ready" before enter active waiting: if phase is not None: Utils.wait_for(lambda: phase.get('start-ready', None) is not None, 0.5, 0.001) phase['configure'] = (True if stream else False) @@ -335,13 +344,14 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __processStartStreamAfterWait(self, *args): + ret = False try: # Wait for the server to start if not self.__waitOnServer(): # pragma: no cover logSys.error("Could not find server, waiting failed") return False # Configure the server - self.__processCmd(*args) + ret = self.__processCmd(*args) except ServerExecutionException as e: # pragma: no cover if self._conf["verbose"] > 1: logSys.exception(e) @@ -350,10 +360,11 @@ class Fail2banClient(Fail2banCmdLine, Thread): "remove " + self._conf["socket"] + ". If " "you used fail2ban-client to start the " "server, adding the -x option will do it") - if self._server: - self._server.quit() - return False - return True + + if not ret and self._server: # stop on error (foreground, config read in another thread): + self._server.quit() + self._server = None + return ret def __waitOnServer(self, alive=True, maxtime=None): if maxtime is None: diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 1268ee9f..c2f6d0be 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -27,15 +27,20 @@ import sys from ..version import version, normVersion from ..protocol import printFormatted -from ..helpers import getLogger, str2LogLevel, getVerbosityFormat +from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError # Gets the instance of the logger. logSys = getLogger("fail2ban") def output(s): # pragma: no cover - print(s) + try: + print(s) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise -CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) +# Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf), +CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket") # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc) PRODUCTION = True @@ -94,9 +99,10 @@ class Fail2banCmdLine(): output("and bans the corresponding IP addresses using firewall rules.") output("") output("Options:") - output(" -c <DIR> configuration directory") - output(" -s <FILE> socket path") - output(" -p <FILE> pidfile path") + output(" -c, --conf <DIR> configuration directory") + output(" -s, --socket <FILE> socket path") + output(" -p, --pidfile <FILE> pidfile path") + output(" --pname <NAME> name of the process (main thread) to identify instance (default fail2ban-server)") output(" --loglevel <LEVEL> logging level") output(" --logtarget <TARGET> logging target, use file-name or stdout, stderr, syslog or sysout.") output(" --syslogsocket auto|<FILE>") @@ -129,17 +135,15 @@ class Fail2banCmdLine(): """ for opt in optList: o = opt[0] - if o == "-c": + if o in ("-c", "--conf"): self._conf["conf"] = opt[1] - elif o == "-s": + elif o in ("-s", "--socket"): self._conf["socket"] = opt[1] - elif o == "-p": + elif o in ("-p", "--pidfile"): self._conf["pidfile"] = opt[1] - elif o.startswith("--log") or o.startswith("--sys"): - self._conf[ o[2:] ] = opt[1] - elif o in ["-d", "--dp", "--dump-pretty"]: + elif o in ("-d", "--dp", "--dump-pretty"): self._conf["dump"] = True if o == "-d" else 2 - elif o == "-t" or o == "--test": + elif o in ("-t", "--test"): self.cleanConfOnly = True self._conf["test"] = True elif o == "-v": @@ -163,12 +167,14 @@ class Fail2banCmdLine(): from ..server.mytime import MyTime output(MyTime.str2seconds(opt[1])) return True - elif o in ["-h", "--help"]: + elif o in ("-h", "--help"): self.dispUsage() return True - elif o in ["-V", "--version"]: + elif o in ("-V", "--version"): self.dispVersion(o == "-V") return True + elif o.startswith("--"): # other long named params (see also resetConf) + self._conf[ o[2:] ] = opt[1] return None def initCmdLine(self, argv): @@ -185,7 +191,8 @@ class Fail2banCmdLine(): try: cmdOpts = 'hc:s:p:xfbdtviqV' cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', - 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] + 'conf=', 'pidfile=', 'pname=', 'socket=', + 'timeout=', 'str2sec=', 'help', 'version', 'dp', 'dump-pretty'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() @@ -227,7 +234,8 @@ class Fail2banCmdLine(): if not conf: self.configurator.readEarly() conf = self.configurator.getEarlyOptions() - self._conf[o] = conf[o] + if o in conf: + self._conf[o] = conf[o] logSys.info("Using socket file %s", self._conf["socket"]) @@ -304,18 +312,24 @@ class Fail2banCmdLine(): # since method is also exposed in API via globally bound variable @staticmethod def _exit(code=0): - if hasattr(os, '_exit') and os._exit: - os._exit(code) - else: - sys.exit(code) + # implicit flush without to produce broken pipe error (32): + sys.stderr.close() + try: + sys.stdout.flush() + # exit: + if hasattr(sys, 'exit') and sys.exit: + sys.exit(code) + else: + os._exit(code) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise @staticmethod def exit(code=0): logSys.debug("Exit with code %s", code) # because of possible buffered output in python, we should flush it before exit: logging.shutdown() - sys.stdout.flush() - sys.stderr.flush() # exit Fail2banCmdLine._exit(code) diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index c81d585e..1f135cf8 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -53,20 +53,30 @@ class Fail2banReader(ConfigReader): opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "syslogsocket", "auto"], + ["string", "allowipv6", "auto"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], + ["int", "dbmaxmatches", None], ["string", "dbpurgeage", "1d"]] self.__opts = ConfigReader.getOptions(self, "Definition", opts) if updateMainOpt: self.__opts.update(updateMainOpt) # check given log-level: str2LogLevel(self.__opts.get('loglevel', 0)) - + # thread options: + opts = [["int", "stacksize", ], + ] + if self.has_section("Thread"): + thopt = ConfigReader.getOptions(self, "Thread", opts) + if thopt: + self.__opts['thread'] = thopt + def convert(self): # Ensure logtarget/level set first so any db errors are captured # Also dbfile should be set before all other database options. # So adding order indices into items, to be stripped after sorting, upon return - order = {"syslogsocket":0, "loglevel":1, "logtarget":2, - "dbfile":50, "dbpurgeage":51} + order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13, + "allowipv6": 14, + "dbfile":50, "dbmaxmatches":51, "dbpurgeage":51} stream = list() for opt in self.__opts: if opt in order: diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 29723dfb..b1795588 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -21,20 +21,25 @@ Fail2Ban reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. This tools can test regular expressions for "fail2ban". - """ __author__ = "Fail2Ban Developers" -__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko" +__copyright__ = """Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors +Copyright of modifications held by their respective authors. +Licensed under the GNU General Public License v2 (GPL). + +Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>. +Many contributions by Yaroslav O. Halchenko, Steven Hiscocks, Sergey G. Brester (sebres).""" + __license__ = "GPL" import getopt import logging +import re import os import shlex import sys import time -import time import urllib from optparse import OptionParser, Option @@ -47,7 +52,7 @@ except ImportError: from ..version import version, normVersion from .filterreader import FilterReader -from ..server.filter import Filter, FileContainer +from ..server.filter import Filter, FileContainer, MyTime from ..server.failregex import Regex, RegexException from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \ @@ -97,33 +102,37 @@ def dumpNormVersion(*args): output(normVersion()) sys.exit(0) -def get_opt_parser(): - # use module docstring for help output - p = OptionParser( - usage="%s [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]\n" % sys.argv[0] + __doc__ - + """ +usage = lambda: "%s [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]" % sys.argv[0] + +class _f2bOptParser(OptionParser): + def format_help(self, *args, **kwargs): + """ Overwritten format helper with full ussage.""" + self.usage = '' + return "Usage: " + usage() + "\n" + __doc__ + """ LOG: - string a string representing a log line - filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal (systemd-python required) + string a string representing a log line + filename path to a log file (/var/log/auth.log) + systemd-journal search systemd journal (systemd-python required), + optionally with backend parameters, see `man jail.conf` + for usage and examples (systemd-journal[journalflags=1]). REGEX: - string a string representing a 'failregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing a 'failregex' + filter name of filter, optionally with options (sshd[mode=aggressive]) + filename path to a filter file (filter.d/sshd.conf) IGNOREREGEX: - string a string representing an 'ignoreregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) +\n""" + OptionParser.format_help(self, *args, **kwargs) + """\n +Report bugs to https://github.com/fail2ban/fail2ban/issues\n +""" + __copyright__ + "\n" -Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors -Copyright of modifications held by their respective authors. -Licensed under the GNU General Public License v2 (GPL). - -Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>. -Many contributions by Yaroslav O. Halchenko and Steven Hiscocks. -Report bugs to https://github.com/fail2ban/fail2ban/issues -""", +def get_opt_parser(): + # use module docstring for help output + p = _f2bOptParser( + usage=usage(), version="%prog " + version) p.add_options([ @@ -160,6 +169,10 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues help="Verbose date patterns/regex in output"), Option("-D", "--debuggex", action='store_true', help="Produce debuggex.com urls for debugging there"), + Option("--no-check-all", action="store_false", dest="checkAllRegex", default=True, + help="Disable check for all regex's"), + Option("-o", "--out", action="store", dest="out", default=None, + help="Set token to print failure information only (row, id, ip, msg, host, ip4, ip6, dns, matches, ...)"), Option("--print-no-missed", action='store_true', help="Do not print any missed lines"), Option("--print-no-ignored", action='store_true', @@ -234,12 +247,15 @@ class Fail2banRegex(object): def __init__(self, opts): # set local protected members from given options: self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.iteritems())) + self._opts = opts self._maxlines_set = False # so we allow to override maxlines in cmdline self._datepattern_set = False self._journalmatch = None self.share_config=dict() self._filter = Filter(None) + self._prefREMatched = 0 + self._prefREGroups = list() self._ignoreregex = list() self._failregex = list() self._time_elapsed = None @@ -253,17 +269,25 @@ class Fail2banRegex(object): self.setJournalMatch(shlex.split(opts.journalmatch)) if opts.timezone: self._filter.setLogTimeZone(opts.timezone) + self._filter.checkFindTime = False + if True: # not opts.out: + MyTime.setAlternateNow(0); # accept every date (years from 19xx up to end of current century, '%ExY' and 'Exy' patterns) + from ..server.strptime import _updateTimeRE + _updateTimeRE() if opts.datepattern: self.setDatePattern(opts.datepattern) if opts.usedns: self._filter.setUseDns(opts.usedns) self._filter.returnRawHost = opts.raw - self._filter.checkFindTime = False - self._filter.checkAllRegex = True - self._opts = opts + self._filter.checkAllRegex = opts.checkAllRegex and not opts.out + # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) + self._filter.ignorePending = bool(opts.out) + # callback to increment ignored RE's by index (during process): + self._filter.onIgnoreRegex = self._onIgnoreRegex + self._backend = 'auto' - def decode_line(self, line): - return FileContainer.decode_line('<LOG>', self._encoding, line) + def output(self, line): + if not self._opts.out: output(line) def encode_line(self, line): return line.encode(self._encoding, 'ignore') @@ -273,43 +297,63 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - output( "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) ) + self.output( "Use datepattern : %s : %s" % ( + pattern, self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: self._filter.setMaxLines(int(v)) self._maxlines_set = True - output( "Use maxlines : %d" % self._filter.getMaxLines() ) + self.output( "Use maxlines : %d" % self._filter.getMaxLines() ) def setJournalMatch(self, v): self._journalmatch = v + def _dumpRealOptions(self, reader, fltOpt): + realopts = {} + combopts = reader.getCombined() + # output all options that are specified in filter-argument as well as some special (mostly interested): + for k in ['logtype', 'datepattern'] + fltOpt.keys(): + # combined options win, but they contain only a sub-set in filter expected keys, + # so get the rest from definition section: + try: + realopts[k] = combopts[k] if k in combopts else reader.get('Definition', k) + except NoOptionError: # pragma: no cover + pass + self.output("Real filter options : %r" % realopts) + def readRegex(self, value, regextype): assert(regextype in ('fail', 'ignore')) regex = regextype + 'regex' # try to check - we've case filter?[options...]?: basedir = self._opts.config + fltName = value fltFile = None fltOpt = {} if regextype == 'fail': - fltName, fltOpt = extractOptions(value) - if fltName is not None: - if "." in fltName[~5:]: - tryNames = (fltName,) - else: - tryNames = (fltName, fltName + '.conf', fltName + '.local') - for fltFile in tryNames: - if not "/" in fltFile: - if os.path.basename(basedir) == 'filter.d': - fltFile = os.path.join(basedir, fltFile) - else: - fltFile = os.path.join(basedir, 'filter.d', fltFile) + if re.search(r'(?ms)^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value): + try: + fltName, fltOpt = extractOptions(value) + if "." in fltName[~5:]: + tryNames = (fltName,) else: - basedir = os.path.dirname(fltFile) - if os.path.isfile(fltFile): - break - fltFile = None + tryNames = (fltName, fltName + '.conf', fltName + '.local') + for fltFile in tryNames: + if not "/" in fltFile: + if os.path.basename(basedir) == 'filter.d': + fltFile = os.path.join(basedir, fltFile) + else: + fltFile = os.path.join(basedir, 'filter.d', fltFile) + else: + basedir = os.path.dirname(fltFile) + if os.path.isfile(fltFile): + break + fltFile = None + except Exception as e: + output("ERROR: Wrong filter name or options: %s" % (str(e),)) + output(" while parsing: %s" % (value,)) + if self._verbose: raise(e) + return False # if it is filter file: if fltFile is not None: if (basedir == self._opts.config @@ -320,13 +364,15 @@ class Fail2banRegex(object): if os.path.basename(basedir) == 'filter.d': basedir = os.path.dirname(basedir) fltName = os.path.splitext(os.path.basename(fltName))[0] - output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) ) + self.output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) ) else: ## foreign file - readexplicit this file and includes if possible: - output( "Use %11s file : %s" % (regex, fltName) ) + self.output( "Use %11s file : %s" % (regex, fltName) ) basedir = None + if not os.path.isabs(fltName): # avoid join with "filter.d" inside FilterReader + fltName = os.path.abspath(fltName) if fltOpt: - output( "Use filter options : %r" % fltOpt ) + self.output( "Use filter options : %r" % fltOpt ) reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir) ret = None try: @@ -342,7 +388,14 @@ class Fail2banRegex(object): if not ret: output( "ERROR: failed to load filter %s" % value ) return False + # set backend-related options (logtype): + reader.applyAutoOptions(self._backend) + # get, interpolate and convert options: reader.getOptions(None) + # show real options if expected: + if self._verbose > 1 or logSys.getEffectiveLevel()<=logging.DEBUG: + self._dumpRealOptions(reader, fltOpt) + # to stream: readercommands = reader.convert() regex_values = {} @@ -384,7 +437,7 @@ class Fail2banRegex(object): return False else: - output( "Use %11s line : %s" % (regex, shortstr(value)) ) + self.output( "Use %11s line : %s" % (regex, shortstr(value)) ) regex_values = {regextype: [RegexStat(value)]} for regextype, regex_values in regex_values.iteritems(): @@ -396,71 +449,144 @@ class Fail2banRegex(object): 'add%sRegex' % regextype.title())(regex.getFailRegex()) return True - def testIgnoreRegex(self, line): - found = False - try: - ret = self._filter.ignoreLine([(line, "", "")]) - if ret is not None: - found = True - regex = self._ignoreregex[ret].inc() - except RegexException as e: # pragma: no cover - output( 'ERROR: %s' % e ) - return False - return found + def _onIgnoreRegex(self, idx, ignoreRegex): + self._lineIgnored = True + self._ignoreregex[idx].inc() def testRegex(self, line, date=None): orgLineBuffer = self._filter._Filter__lineBuffer + # duplicate line buffer (list can be changed inplace during processLine): + if self._filter.getMaxLines() > 1: + orgLineBuffer = orgLineBuffer[:] fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() - is_ignored = False + is_ignored = self._lineIgnored = False try: found = self._filter.processLine(line, date) lines = [] - line = self._filter.processedLine() ret = [] for match in found: - # Append True/False flag depending if line was matched by - # more than one regex - match.append(len(ret)>1) - regex = self._failregex[match[0]] - regex.inc() - regex.appendIP(match) + if not self._opts.out: + # Append True/False flag depending if line was matched by + # more than one regex + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) if not match[3].get('nofail'): ret.append(match) else: is_ignored = True + if self._opts.out: # (formated) output - don't need stats: + return None, ret, None + # prefregex stats: + if self._filter.prefRegex: + pre = self._filter.prefRegex + if pre.hasMatched(): + self._prefREMatched += 1 + if self._verbose: + if len(self._prefREGroups) < self._maxlines: + self._prefREGroups.append(pre.getGroups()) + else: + if len(self._prefREGroups) == self._maxlines: + self._prefREGroups.append('...') except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) - return False - for bufLine in orgLineBuffer[int(fullBuffer):]: - if bufLine not in self._filter._Filter__lineBuffer: - try: - self._line_stats.missed_lines.pop( - self._line_stats.missed_lines.index("".join(bufLine))) - if self._debuggex: - self._line_stats.missed_lines_timeextracted.pop( - self._line_stats.missed_lines_timeextracted.index( - "".join(bufLine[::2]))) - except ValueError: - pass - # if buffering - add also another lines from match: - if self._print_all_matched: - if not self._debuggex: - self._line_stats.matched_lines.append("".join(bufLine)) - else: - lines.append(bufLine[0] + bufLine[2]) - self._line_stats.matched += 1 - self._line_stats.missed -= 1 + return None, 0, None + if self._filter.getMaxLines() > 1: + for bufLine in orgLineBuffer[int(fullBuffer):]: + if bufLine not in self._filter._Filter__lineBuffer: + try: + self._line_stats.missed_lines.pop( + self._line_stats.missed_lines.index("".join(bufLine))) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.pop( + self._line_stats.missed_lines_timeextracted.index( + "".join(bufLine[::2]))) + except ValueError: + pass + # if buffering - add also another lines from match: + if self._print_all_matched: + if not self._debuggex: + self._line_stats.matched_lines.append("".join(bufLine)) + else: + lines.append(bufLine[0] + bufLine[2]) + self._line_stats.matched += 1 + self._line_stats.missed -= 1 if lines: # pre-lines parsed in multiline mode (buffering) - lines.append(line) + lines.append(self._filter.processedLine()) line = "\n".join(lines) - return line, ret, is_ignored + return line, ret, (is_ignored or self._lineIgnored) + + def _prepaireOutput(self): + """Prepares output- and fetch-function corresponding given '--out' option (format)""" + ofmt = self._opts.out + if ofmt in ('id', 'fid'): + def _out(ret): + for r in ret: + output(r[1]) + elif ofmt == 'ip': + def _out(ret): + for r in ret: + output(r[3].get('ip', r[1])) + elif ofmt == 'msg': + def _out(ret): + for r in ret: + for r in r[3].get('matches'): + if not isinstance(r, basestring): + r = ''.join(r for r in r) + output(r) + elif ofmt == 'row': + def _out(ret): + for r in ret: + output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches'))) + elif '<' not in ofmt: + def _out(ret): + for r in ret: + output(r[3].get(ofmt)) + else: # extended format with tags substitution: + from ..server.actions import Actions, CommandAction, BanTicket + def _escOut(t, v): + # use safe escape (avoid inject on pseudo tag "\x00msg\x00"): + if t not in ('msg',): + return v.replace('\x00', '\\x00') + return v + def _out(ret): + rows = [] + wrap = {'NL':0} + for r in ret: + ticket = BanTicket(r[1], time=r[2], data=r[3]) + aInfo = Actions.ActionInfo(ticket) + # if msg tag is used - output if single line (otherwise let it as is to wrap multilines later): + def _get_msg(self): + if not wrap['NL'] and len(r[3].get('matches', [])) <= 1: + return self['matches'] + else: # pseudo tag for future replacement: + wrap['NL'] = 1 + return "\x00msg\x00" + aInfo['msg'] = _get_msg + # not recursive interpolation (use safe escape): + v = CommandAction.replaceDynamicTags(ofmt, aInfo, escapeVal=_escOut) + if wrap['NL']: # contains multiline tags (msg): + rows.append((r, v)) + continue + output(v) + # wrap multiline tag (msg) interpolations to single line: + for r, v in rows: + for r in r[3].get('matches'): + if not isinstance(r, basestring): + r = ''.join(r for r in r) + r = v.replace("\x00msg\x00", r) + output(r) + return _out + def process(self, test_lines): t0 = time.time() + if self._opts.out: # get out function + out = self._prepaireOutput() for line in test_lines: if isinstance(line, tuple): - line_datetimestripped, ret, is_ignored = self.testRegex( - line[0], line[1]) + line_datetimestripped, ret, is_ignored = self.testRegex(line[0], line[1]) line = "".join(line[0]) else: line = line.rstrip('\r\n') @@ -468,8 +594,10 @@ class Fail2banRegex(object): # skip comment and empty lines continue line_datetimestripped, ret, is_ignored = self.testRegex(line) - if not is_ignored: - is_ignored = self.testIgnoreRegex(line_datetimestripped) + + if self._opts.out: # (formated) output: + if len(ret) > 0 and not is_ignored: out(ret) + continue if is_ignored: self._line_stats.ignored += 1 @@ -477,28 +605,25 @@ class Fail2banRegex(object): self._line_stats.ignored_lines.append(line) if self._debuggex: self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) - - if len(ret) > 0: - assert(not is_ignored) + elif len(ret) > 0: self._line_stats.matched += 1 if self._print_all_matched: self._line_stats.matched_lines.append(line) if self._debuggex: self._line_stats.matched_lines_timeextracted.append(line_datetimestripped) else: - if not is_ignored: - self._line_stats.missed += 1 - if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): - self._line_stats.missed_lines.append(line) - if self._debuggex: - self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) + self._line_stats.missed += 1 + if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): + self._line_stats.missed_lines.append(line) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 self._time_elapsed = time.time() - t0 def printLines(self, ltype): lstats = self._line_stats - assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) + assert(lstats.missed == lstats.tested - (lstats.matched + lstats.ignored)) lines = lstats[ltype] l = lstats[ltype + '_lines'] multiline = self._filter.getMaxLines() > 1 @@ -528,6 +653,7 @@ class Fail2banRegex(object): "to print all %d lines" % (header, ltype, lines) ) def printStats(self): + if self._opts.out: return True output( "" ) output( "Results" ) output( "=======" ) @@ -555,7 +681,18 @@ class Fail2banRegex(object): pprint_list(out, " #) [# of hits] regular expression") return total - # Print title + # Print prefregex: + if self._filter.prefRegex: + #self._filter.prefRegex.hasMatched() + pre = self._filter.prefRegex + out = [pre.getRegex()] + if self._verbose: + for grp in self._prefREGroups: + out.append(" %s" % (grp,)) + output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) ) + pprint_list(out) + + # Print regex's: total = print_failregexes("Failregex", self._failregex) _ = print_failregexes("Ignoreregex", self._ignoreregex) @@ -587,14 +724,13 @@ class Fail2banRegex(object): return True - def file_lines_gen(self, hdlr): - for line in hdlr: - yield self.decode_line(line) - def start(self, args): cmd_log, cmd_regex = args[:2] + if cmd_log.startswith("systemd-journal"): # pragma: no cover + self._backend = 'systemd' + try: if not self.readRegex(cmd_regex, 'fail'): # pragma: no cover return False @@ -606,10 +742,10 @@ class Fail2banRegex(object): if os.path.isfile(cmd_log): try: - hdlr = open(cmd_log, 'rb') - output( "Use log file : %s" % cmd_log ) - output( "Use encoding : %s" % self._encoding ) - test_lines = self.file_lines_gen(hdlr) + test_lines = FileContainer(cmd_log, self._encoding, doOpen=True) + + self.output( "Use log file : %s" % cmd_log ) + self.output( "Use encoding : %s" % self._encoding ) except IOError as e: # pragma: no cover output( e ) return False @@ -617,8 +753,8 @@ class Fail2banRegex(object): if not FilterSystemd: output( "Error: systemd library not found. Exiting..." ) return False - output( "Use systemd journal" ) - output( "Use encoding : %s" % self._encoding ) + self.output( "Use systemd journal" ) + self.output( "Use encoding : %s" % self._encoding ) backend, beArgs = extractOptions(cmd_log) flt = FilterSystemd(None, **beArgs) flt.setLogEncoding(self._encoding) @@ -627,23 +763,23 @@ class Fail2banRegex(object): self.setDatePattern(None) if journalmatch: flt.addJournalMatch(journalmatch) - output( "Use journal match : %s" % " ".join(journalmatch) ) + self.output( "Use journal match : %s" % " ".join(journalmatch) ) test_lines = journal_lines_gen(flt, myjournal) else: # if single line parsing (without buffering) - if self._filter.getMaxLines() <= 1: - output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) + if self._filter.getMaxLines() <= 1 and '\n' not in cmd_log: + self.output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) test_lines = [ cmd_log ] - else: # multi line parsing (with buffering) + else: # multi line parsing (with and without buffering) test_lines = cmd_log.split("\n") - output( "Use multi line : %s line(s)" % len(test_lines) ) + self.output( "Use multi line : %s line(s)" % len(test_lines) ) for i, l in enumerate(test_lines): if i >= 5: - output( "| ..." ); break - output( "| %2.2s: %s" % (i+1, shortstr(l)) ) - output( "`-" ) + self.output( "| ..." ); break + self.output( "| %2.2s: %s" % (i+1, shortstr(l)) ) + self.output( "`-" ) - output( "" ) + self.output( "" ) self.process(test_lines) @@ -654,6 +790,7 @@ class Fail2banRegex(object): def exec_command_line(*args): + logging.exitOnIOError = True parser = get_opt_parser() (opts, args) = parser.parse_args(*args) errors = [] @@ -666,14 +803,15 @@ def exec_command_line(*args): if not len(args) in (2, 3): errors.append("ERROR: provide both <LOG> and <REGEX>.") if errors: - sys.stderr.write("\n".join(errors) + "\n\n") parser.print_help() + sys.stderr.write("\n" + "\n".join(errors) + "\n") sys.exit(255) - output( "" ) - output( "Running tests" ) - output( "=============" ) - output( "" ) + if not opts.out: + output( "" ) + output( "Running tests" ) + output( "=============" ) + output( "" ) # Log level (default critical): opts.log_level = str2LogLevel(opts.log_level) @@ -694,6 +832,14 @@ def exec_command_line(*args): stdout.setFormatter(Formatter(getVerbosityFormat(opts.verbose, fmt))) logSys.addHandler(stdout) - fail2banRegex = Fail2banRegex(opts) + try: + fail2banRegex = Fail2banRegex(opts) + except Exception as e: + if opts.verbose or logSys.getEffectiveLevel()<=logging.DEBUG: + logSys.critical(e, exc_info=True) + else: + output( 'ERROR: %s' % e ) + sys.exit(255) + if not fail2banRegex.start(args): sys.exit(255) diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index d94d13ff..eee78d5f 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -44,7 +44,7 @@ class Fail2banServer(Fail2banCmdLine): # Start the Fail2ban server in background/foreground (daemon mode or not). @staticmethod - def startServerDirect(conf, daemon=True): + def startServerDirect(conf, daemon=True, setServer=None): logSys.debug(" direct starting of server in %s, deamon: %s", os.getpid(), daemon) from ..server.server import Server server = None @@ -52,6 +52,10 @@ class Fail2banServer(Fail2banCmdLine): # Start it in foreground (current thread, not new process), # server object will internally fork self if daemon is True server = Server(daemon) + # notify caller - set server handle: + if setServer: + setServer(server) + # run: server.start(conf["socket"], conf["pidfile"], conf["force"], conf=conf) @@ -63,6 +67,10 @@ class Fail2banServer(Fail2banCmdLine): if conf["verbose"] > 1: logSys.exception(e2) raise + finally: + # notify waiting thread server ready resp. done (background execution, error case, etc): + if conf.get('onstart'): + conf['onstart']() return server @@ -179,27 +187,15 @@ class Fail2banServer(Fail2banCmdLine): # Start new thread with client to read configuration and # transfer it to the server: cli = self._Fail2banClient() + cli._conf = self._conf phase = dict() logSys.debug('Configure via async client thread') cli.configureServer(phase=phase) - # wait, do not continue if configuration is not 100% valid: - Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"], 0.001) - logSys.log(5, ' server phase %s', phase) - if not phase.get('start', False): - raise ServerExecutionException('Async configuration of server failed') - # event for server ready flag: - def _server_ready(): - phase['start-ready'] = True - logSys.log(5, ' server phase %s', phase) - # notify waiting thread if server really ready - self._conf['onstart'] = _server_ready # Start server, daemonize it, etc. pid = os.getpid() - server = Fail2banServer.startServerDirect(self._conf, background) - # notify waiting thread server ready resp. done (background execution, error case, etc): - if not nonsync: - _server_ready() + server = Fail2banServer.startServerDirect(self._conf, background, + cli._set_server if cli else None) # If forked - just exit other processes if pid != os.getpid(): # pragma: no cover os._exit(0) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 9edeb2f3..24341014 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -37,9 +37,10 @@ logSys = getLogger(__name__) class FilterReader(DefinitionInitConfigReader): _configOpts = { + "usedns": ["string", None], "prefregex": ["string", None], "ignoreregex": ["string", None], - "failregex": ["string", ""], + "failregex": ["string", None], "maxlines": ["int", None], "datepattern": ["string", None], "journalmatch": ["string", None], @@ -52,35 +53,48 @@ class FilterReader(DefinitionInitConfigReader): def getFile(self): return self.__file + def applyAutoOptions(self, backend): + # set init option to backend-related logtype, considering + # that the filter settings may be overwritten in its local: + if (not self._initOpts.get('logtype') and + not self.has_option('Definition', 'logtype', False) + ): + self._initOpts['logtype'] = ['file','journal'][int(backend.startswith("systemd"))] + def convert(self): stream = list() opts = self.getCombined() if not len(opts): return stream + return FilterReader._fillStream(stream, opts, self._jailName) + + @staticmethod + def _fillStream(stream, opts, jailName): + prio0idx = 0 for opt, value in opts.iteritems(): + # Do not send a command if the value is not set (empty). + if value is None: continue if opt in ("failregex", "ignoreregex"): - if value is None: continue multi = [] for regex in value.split('\n'): # Do not send a command if the rule is empty. if regex != '': multi.append(regex) if len(multi) > 1: - stream.append(["multi-set", self._jailName, "add" + opt, multi]) + stream.append(["multi-set", jailName, "add" + opt, multi]) elif len(multi): - stream.append(["set", self._jailName, "add" + opt, multi[0]]) - elif opt in ('maxlines', 'prefregex'): - # Be sure we set this options first. - stream.insert(0, ["set", self._jailName, opt, value]) + stream.append(["set", jailName, "add" + opt, multi[0]]) + elif opt in ('usedns', 'maxlines', 'prefregex'): + # Be sure we set this options first, and usedns is before all regex(s). + stream.insert(0 if opt == 'usedns' else prio0idx, + ["set", jailName, opt, value]) + prio0idx += 1 elif opt in ('datepattern'): - stream.append(["set", self._jailName, opt, value]) - # Do not send a command if the match is empty. + stream.append(["set", jailName, opt, value]) elif opt == 'journalmatch': - if value is None: continue for match in value.split("\n"): if match == '': continue stream.append( - ["set", self._jailName, "addjournalmatch"] + - shlex.split(match)) + ["set", jailName, "addjournalmatch"] + shlex.split(match)) return stream diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 4ee97317..37746d4c 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -33,7 +33,7 @@ from .configreader import ConfigReaderUnshared, ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader from ..version import version -from ..helpers import getLogger, extractOptions, splitwords +from ..helpers import getLogger, extractOptions, splitWithOptions, splitwords # Gets the instance of the logger. logSys = getLogger(__name__) @@ -86,43 +86,54 @@ class JailReader(ConfigReader): logSys.warning("File %s is a dangling link, thus cannot be monitored" % p) return pathList + _configOpts1st = { + "enabled": ["bool", False], + "backend": ["string", "auto"], + "filter": ["string", ""] + } + _configOpts = { + "enabled": ["bool", False], + "backend": ["string", "auto"], + "maxretry": ["int", None], + "maxmatches": ["int", None], + "findtime": ["string", None], + "bantime": ["string", None], + "bantime.increment": ["bool", None], + "bantime.factor": ["string", None], + "bantime.formula": ["string", None], + "bantime.multipliers": ["string", None], + "bantime.maxtime": ["string", None], + "bantime.rndtime": ["string", None], + "bantime.overalljails": ["bool", None], + "ignorecommand": ["string", None], + "ignoreself": ["bool", None], + "ignoreip": ["string", None], + "ignorecache": ["string", None], + "filter": ["string", ""], + "logtimezone": ["string", None], + "logencoding": ["string", None], + "logpath": ["string", None], + "action": ["string", ""] + } + _configOpts.update(FilterReader._configOpts) + + _ignoreOpts = set(['action', 'filter', 'enabled'] + FilterReader._configOpts.keys()) + def getOptions(self): - opts1st = [["bool", "enabled", False], - ["string", "filter", ""]] - opts = [["bool", "enabled", False], - ["string", "backend", "auto"], - ["int", "maxretry", None], - ["string", "findtime", None], - ["string", "bantime", None], - ["bool", "bantime.increment", None], - ["string", "bantime.factor", None], - ["string", "bantime.formula", None], - ["string", "bantime.multipliers", None], - ["string", "bantime.maxtime", None], - ["string", "bantime.rndtime", None], - ["bool", "bantime.overalljails", None], - ["string", "usedns", None], # be sure usedns is before all regex(s) in stream - ["string", "failregex", None], - ["string", "ignoreregex", None], - ["string", "ignorecommand", None], - ["bool", "ignoreself", None], - ["string", "ignoreip", None], - ["string", "ignorecache", None], - ["string", "filter", ""], - ["string", "datepattern", None], - ["string", "logtimezone", None], - ["string", "logencoding", None], - ["string", "logpath", None], # logpath after all log-related data (backend, date-pattern, etc) - ["string", "action", ""]] + + basedir = self.getBaseDir() # Before interpolation (substitution) add static options always available as default: - defsec = self._cfg.get_defaults() - defsec["fail2ban_version"] = version + self.merge_defaults({ + "fail2ban_version": version, + "fail2ban_confpath": basedir + }) try: # Read first options only needed for merge defaults ('known/...' from filter): - self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True) + self.__opts = ConfigReader.getOptions(self, self.__name, self._configOpts1st, + shouldExist=True) if not self.__opts: # pragma: no cover raise JailDefError("Init jail options failed") @@ -132,15 +143,18 @@ class JailReader(ConfigReader): # Read filter flt = self.__opts["filter"] if flt: - filterName, filterOpt = extractOptions(flt) - if not filterName: - raise JailDefError("Invalid filter definition %r" % flt) + try: + filterName, filterOpt = extractOptions(flt) + except ValueError as e: + raise JailDefError("Invalid filter definition %r: %s" % (flt, e)) self.__filter = FilterReader( filterName, self.__name, filterOpt, - share_config=self.share_config, basedir=self.getBaseDir()) + share_config=self.share_config, basedir=basedir) ret = self.__filter.read() if not ret: raise JailDefError("Unable to read the filter %r" % filterName) + # set backend-related options (logtype): + self.__filter.applyAutoOptions(self.__opts.get('backend', '')) # merge options from filter as 'known/...' (all options unfiltered): self.__filter.getOptions(self.__opts, all=True) ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') @@ -149,7 +163,7 @@ class JailReader(ConfigReader): logSys.warning("No filter set for jail %s" % self.__name) # Read second all options (so variables like %(known/param) can be interpolated): - self.__opts = ConfigReader.getOptions(self, self.__name, opts) + self.__opts = ConfigReader.getOptions(self, self.__name, self._configOpts) if not self.__opts: # pragma: no cover raise JailDefError("Read jail options failed") @@ -158,13 +172,16 @@ class JailReader(ConfigReader): self.__filter.getOptions(self.__opts) # Read action - for act in self.__opts["action"].split('\n'): + for act in splitWithOptions(self.__opts["action"]): try: + act = act.strip() if not act: # skip empty actions continue - actName, actOpt = extractOptions(act) - if not actName: - raise JailDefError("Invalid action definition %r" % act) + # join with previous line if needed (consider possible new-line): + try: + actName, actOpt = extractOptions(act) + except ValueError as e: + raise JailDefError("Invalid action definition %r: %s" % (act, e)) if actName.endswith(".py"): self.__actions.append([ "set", @@ -172,13 +189,13 @@ class JailReader(ConfigReader): "addaction", actOpt.pop("actname", os.path.splitext(actName)[0]), os.path.join( - self.getBaseDir(), "action.d", actName), + basedir, "action.d", actName), json.dumps(actOpt), ]) else: action = ActionReader( actName, self.__name, actOpt, - share_config=self.share_config, basedir=self.getBaseDir()) + share_config=self.share_config, basedir=basedir) ret = action.read() if ret: action.getOptions(self.__opts) @@ -213,15 +230,19 @@ class JailReader(ConfigReader): """ stream = [] + stream2 = [] e = self.__opts.get('config-error') if e: stream.extend([['config-error', "Jail '%s' skipped, because of wrong configuration: %s" % (self.__name, e)]]) return stream + # fill jail with filter options, using filter (only not overriden in jail): if self.__filter: stream.extend(self.__filter.convert()) + # and using options from jail: + FilterReader._fillStream(stream, self.__opts, self.__name) for opt, value in self.__opts.iteritems(): if opt == "logpath": - if self.__opts.get('backend', None).startswith("systemd"): continue + if self.__opts.get('backend', '').startswith("systemd"): continue found_files = 0 for path in value.split("\n"): path = path.rsplit(" ", 1) @@ -231,33 +252,22 @@ class JailReader(ConfigReader): logSys.notice("No file(s) found for glob %s" % path) for p in pathList: found_files += 1 - stream.append( + # logpath after all log-related data (backend, date-pattern, etc) + stream2.append( ["set", self.__name, "addlogpath", p, tail]) if not found_files: msg = "Have not found any log file for %s jail" % self.__name if not allow_no_files: raise ValueError(msg) logSys.warning(msg) - - elif opt == "logencoding": - stream.append(["set", self.__name, "logencoding", value]) elif opt == "backend": backend = value elif opt == "ignoreip": - for ip in splitwords(value): - stream.append(["set", self.__name, "addignoreip", ip]) - elif opt in ("failregex", "ignoreregex"): - multi = [] - for regex in value.split('\n'): - # Do not send a command if the rule is empty. - if regex != '': - multi.append(regex) - if len(multi) > 1: - stream.append(["multi-set", self.__name, "add" + opt, multi]) - elif len(multi): - stream.append(["set", self.__name, "add" + opt, multi[0]]) - elif opt not in ('action', 'filter', 'enabled'): + stream.append(["set", self.__name, "addignoreip"] + splitwords(value)) + elif opt not in JailReader._ignoreOpts: stream.append(["set", self.__name, opt, value]) + # consider options order (after other options): + if stream2: stream += stream2 for action in self.__actions: if isinstance(action, (ConfigReaderUnshared, ConfigReader)): stream.extend(action.convert()) |